http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-cassandra-server-template.xml ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-cassandra-server-template.xml b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-cassandra-server-template.xml index f85dcd9..03b3346 100644 --- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-cassandra-server-template.xml +++ b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-cassandra-server-template.xml @@ -25,7 +25,11 @@ http://www.springframework.org/schema/util/spring-util.xsd";> <!-- Cassandra connection settings --> - <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.RoundRobinPolicy"/> + <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.TokenAwarePolicy"> + <constructor-arg type="com.datastax.driver.core.policies.LoadBalancingPolicy"> + <bean class="com.datastax.driver.core.policies.RoundRobinPolicy"/> + </constructor-arg> + </bean> <util:list id="contactPoints" value-type="java.lang.String"> ${CASSANDRA_SEEDS} @@ -108,15 +112,14 @@ <property name="cacheConfiguration"> <list> - <!-- Partitioned cache example configuration (Atomic mode). --> - <bean class="org.apache.ignite.configuration.CacheConfiguration"> - <property name="atomicityMode" value="ATOMIC"/> - <property name="backups" value="1"/> - </bean> - <!-- Configuring persistence for "cache1" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache1"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/> @@ -131,6 +134,11 @@ <!-- Configuring persistence for "cache2" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache2"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/> @@ -145,6 +153,11 @@ <!-- Configuring persistence for "cache3" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache3"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/>
http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh index e0643b7..bfe3371 100644 --- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh +++ b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-env.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +17,10 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Environment setup script from Ignite +# ----------------------------------------------------------------------------------------------- + JVM_OPTS="-Xms10g -Xmx10g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m" JVM_OPTS="$JVM_OPTS -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseTLAB -XX:NewSize=128m -XX:MaxNewSize=768m" #JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=0 -XX:SurvivorRatio=1024 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60" http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh index bb1ff0c..f2c1557 100644 --- a/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh +++ b/modules/cassandra/src/test/bootstrap/aws/ignite/ignite-start.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,380 +17,70 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Script to start Ignite daemon (used by ignite-bootstrap.sh) +# ----------------------------------------------------------------------------------------------- + #profile=/home/ignite/.bash_profile profile=/root/.bash_profile . $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "ignite" -terminate() -{ - if [[ "$S3_BOOTSTRAP_SUCCESS_URL" != */ ]]; then - S3_BOOTSTRAP_SUCCESS_URL=${S3_BOOTSTRAP_SUCCESS_URL}/ - fi - - if [[ "$S3_BOOTSTRAP_FAILURE_URL" != */ ]]; then - S3_BOOTSTRAP_FAILURE_URL=${S3_BOOTSTRAP_FAILURE_URL}/ - fi - - msg=$HOST_NAME - - if [ -n "$1" ]; then - echo "[ERROR] $1" - echo "[ERROR]-----------------------------------------------------" - echo "[ERROR] Failed to start Ignite node" - echo "[ERROR]-----------------------------------------------------" - msg=$1 - reportFolder=${S3_BOOTSTRAP_FAILURE_URL}${HOST_NAME} - reportFile=$reportFolder/__error__ - else - echo "[INFO]-----------------------------------------------------" - echo "[INFO] Ignite node successfully started" - echo "[INFO]-----------------------------------------------------" - reportFolder=${S3_BOOTSTRAP_SUCCESS_URL}${HOST_NAME} - reportFile=$reportFolder/__success__ - fi - - echo $msg > /opt/ignite/start_result - - aws s3 rm --recursive $reportFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed drop report folder: $reportFolder" - fi - - if [ -d "/opt/ignite/work/log" ]; then - aws s3 sync --sse AES256 /opt/ignite/work/log $reportFolder - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to export Ignite logs to: $reportFolder" - fi - fi - - aws s3 cp --sse AES256 /opt/ignite/start_result $reportFile - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to export node start result to: $reportFile" - fi - - rm -f /opt/ignite/start_result /opt/ignite/join-lock /opt/ignite/remote-join-lock - - if [ -n "$1" ]; then - exit 1 - fi - - exit 0 -} - -registerNode() -{ - echo "[INFO] Registering Ignite node seed: ${S3_IGNITE_NODES_DISCOVERY_URL}$HOST_NAME" - - aws s3 cp --sse AES256 /opt/ignite/join-lock ${S3_IGNITE_NODES_DISCOVERY_URL}$HOST_NAME - if [ $? -ne 0 ]; then - terminate "Failed to register Ignite node seed: ${S3_IGNITE_NODES_DISCOVERY_URL}$HOST_NAME" - fi - - echo "[INFO] Ignite node seed successfully registered" -} - -unregisterNode() -{ - echo "[INFO] Removing Ignite node registration from: ${S3_IGNITE_NODES_DISCOVERY_URL}$HOST_NAME" - aws s3 rm ${S3_IGNITE_NODES_DISCOVERY_URL}$HOST_NAME - echo "[INFO] Ignite node registration removed" -} - -cleanupMetadata() -{ - echo "[INFO] Running cleanup" - aws s3 rm $S3_IGNITE_NODES_JOIN_LOCK_URL - aws s3 rm --recursive $S3_IGNITE_NODES_DISCOVERY_URL - aws s3 rm --recursive $S3_BOOTSTRAP_SUCCESS_URL - aws s3 rm --recursive $S3_BOOTSTRAP_FAILURE_URL - echo "[INFO] Cleanup completed" -} - +# Setups Cassandra seeds for this Ignite node being able to connect to Cassandra. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. setupCassandraSeeds() { - echo "[INFO] Setting up Cassandra seeds" - - echo "[INFO] Looking for Cassandra seeds in: $S3_CASSANDRA_NODES_DISCOVERY_URL" - - startTime=$(date +%s) - - while true; do - seeds=$(aws s3 ls $S3_CASSANDRA_NODES_DISCOVERY_URL | grep -v PRE | sed -r "s/^.* //g") - if [ -n "$seeds" ]; then - seeds=($seeds) - length=${#seeds[@]} - - if [ $length -lt 4 ]; then - seed1=${seeds[0]} - seed2=${seeds[1]} - seed3=${seeds[2]} - else - pos1=$(($RANDOM%$length)) - pos2=$(($RANDOM%$length)) - pos3=$(($RANDOM%$length)) - seed1=${seeds[${pos1}]} - seed2=${seeds[${pos2}]} - seed3=${seeds[${pos3}]} - fi - - CASSANDRA_SEEDS="<value>$seed1<\/value>" - - if [ "$seed2" != "$seed1" ] && [ -n "$seed2" ]; then - CASSANDRA_SEEDS="$CASSANDRA_SEEDS<value>$seed2<\/value>" - fi - - if [ "$seed3" != "$seed2" ] && [ "$seed3" != "$seed1" ] && [ -n "$seed3" ]; then - CASSANDRA_SEEDS="$CASSANDRA_SEEDS<value>$seed3<\/value>" - fi - - echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS" + setupClusterSeeds "cassandra" "true" - cat /opt/ignite/config/ignite-cassandra-server-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server.xml + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} - return 0 - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $NODE_STARTUP_TIME ]; then - terminate "${NODE_STARTUP_TIME}min timeout expired, but no Cassandra nodes is up and running" - fi + CASSANDRA_SEEDS= - echo "[INFO] Waiting for the first Cassandra node to start and publish its seed, time passed ${duration}min" + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + CASSANDRA_SEEDS="${CASSANDRA_SEEDS}<value>$seed<\/value>" + done - sleep 1m - done + cat /opt/ignite/config/ignite-cassandra-server-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server.xml } +# Setups Ignite nodes which this EC2 Ignite node will use to send its metadata and join Ignite cluster setupIgniteSeeds() { - echo "[INFO] Setting up Ignite seeds" - - if [ "$FIRST_NODE" == "true" ]; then - IGNITE_SEEDS="<value>127.0.0.1:47500..47509<\/value>" - echo "[INFO] Using localhost address as a seed for the first Ignite node: $IGNITE_SEEDS" - aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY_URL::-1} - if [ $? -ne 0 ]; then - terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY_URL" - fi - - cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml - mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml + if [ "$FIRST_NODE_LOCK" == "true" ]; then + echo "[INFO] Setting up Ignite seeds" - return 0 - fi + CLUSTER_SEEDS="127.0.0.1:47500..47509" - echo "[INFO] Looking for Ignite seeds in: $S3_IGNITE_NODES_DISCOVERY_URL" - - startTime=$(date +%s) - - while true; do - seeds=$(aws s3 ls $S3_IGNITE_NODES_DISCOVERY_URL | grep -v PRE | sed -r "s/^.* //g") - if [ -n "$seeds" ]; then - seeds=($seeds) - length=${#seeds[@]} - - if [ $length -lt 4 ]; then - seed1=${seeds[0]} - seed2=${seeds[1]} - seed3=${seeds[2]} - else - pos1=$(($RANDOM%$length)) - pos2=$(($RANDOM%$length)) - pos3=$(($RANDOM%$length)) - seed1=${seeds[${pos1}]} - seed2=${seeds[${pos2}]} - seed3=${seeds[${pos3}]} - fi + echo "[INFO] Using localhost address as a seed for the first Ignite node: $CLUSTER_SEEDS" - IGNITE_SEEDS="<value>$seed1<\/value>" - - if [ "$seed2" != "$seed1" ] && [ -n "$seed2" ]; then - IGNITE_SEEDS="$IGNITE_SEEDS<value>$seed2<\/value>" - fi - - if [ "$seed3" != "$seed2" ] && [ "$seed3" != "$seed1" ] && [ -n "$seed3" ]; then - IGNITE_SEEDS="$IGNITE_SEEDS<value>$seed3<\/value>" - fi - - echo "[INFO] Using Ignite seeds: $IGNITE_SEEDS" - - cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml - mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml - - return 0 - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $NODE_STARTUP_TIME ]; then - terminate "${NODE_STARTUP_TIME}min timeout expired, but no Ignite nodes is up and running" + aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} + if [ $? -ne 0 ]; then + terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" fi - - echo "[INFO] Waiting for the first Ignite node to start and publish its seed, time passed ${duration}min" - - sleep 1m - done -} - -tryToGetFirstNodeLock() -{ - echo "[INFO] Trying to get first node lock" - - checkFirstNodeLockExist - if [ $? -ne 0 ]; then - return 1 - fi - - createFirstNodeLock - - sleep 5s - - rm -Rf /opt/ignite/first-node-lock - - aws s3 cp $S3_IGNITE_FIRST_NODE_LOCK_URL /opt/ignite/first-node-lock - if [ $? -ne 0 ]; then - echo "[WARN] Failed to check just created first node lock" - return 1 - fi - - first_host=$(cat /opt/ignite/first-node-lock) - - rm -f /opt/ignite/first-node-lock - - if [ "$first_host" != "$HOST_NAME" ]; then - echo "[INFO] Node $first_host has discarded previously created first node lock" - return 1 - fi - - echo "[INFO] Congratulations, got first node lock" - - return 0 -} - -checkFirstNodeLockExist() -{ - echo "[INFO] Checking for the first node lock" - - lockExists=$(aws s3 ls $S3_IGNITE_FIRST_NODE_LOCK_URL) - if [ -n "$lockExists" ]; then - echo "[INFO] First node lock already exists" - return 1 - fi - - echo "[INFO] First node lock doesn't exist yet" - - return 0 -} - -createFirstNodeLock() -{ - aws s3 cp --sse AES256 /opt/ignite/join-lock $S3_IGNITE_FIRST_NODE_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to create first node lock" - fi - echo "[INFO] Created first node lock" -} - -removeFirstNodeLock() -{ - aws s3 rm $S3_IGNITE_FIRST_NODE_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to remove first node lock" - fi - echo "[INFO] Removed first node lock" -} - -tryToGetClusterJoinLock() -{ - echo "[INFO] Trying to get cluster join lock" - - checkClusterJoinLockExist - if [ $? -ne 0 ]; then - return 1 - fi - - createClusterJoinLock - - sleep 5s - - rm -Rf /opt/ignite/remote-join-lock - - aws s3 cp $S3_IGNITE_NODES_JOIN_LOCK_URL /opt/ignite/remote-join-lock - if [ $? -ne 0 ]; then - echo "[WARN] Failed to check just created cluster join lock" - return 1 - fi - - join_host=$(cat /opt/ignite/remote-join-lock) - - if [ "$join_host" != "$HOST_NAME" ]; then - echo "[INFO] Node $first_host has discarded previously created cluster join lock" - return 1 - fi - - echo "[INFO] Congratulations, got cluster join lock" - - return 0 -} - -checkClusterJoinLockExist() -{ - echo "[INFO] Checking for the cluster join lock" - - lockExists=$(aws s3 ls $S3_IGNITE_NODES_JOIN_LOCK_URL) - if [ -n "$lockExists" ]; then - echo "[INFO] Cluster join lock already exists" - return 1 - fi - - echo "[INFO] Cluster join lock doesn't exist" - - return 0 -} - -createClusterJoinLock() -{ - aws s3 cp --sse AES256 /opt/ignite/join-lock $S3_IGNITE_NODES_JOIN_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to create cluster join lock" + else + setupClusterSeeds "ignite" "true" fi - echo "[INFO] Created cluster join lock" -} -removeClusterJoinLock() -{ - aws s3 rm $S3_IGNITE_NODES_JOIN_LOCK_URL - if [ $? -ne 0 ]; then - terminate "Failed to remove cluster join lock" - fi - echo "[INFO] Removed cluster join lock" -} + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} -waitToJoinIgniteCluster() -{ - echo "[INFO] Waiting to join Ignite cluster" + IGNITE_SEEDS= - while true; do - tryToGetClusterJoinLock + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" + done - if [ $? -ne 0 ]; then - echo "[INFO] Another node is trying to join cluster. Waiting for extra 1min." - sleep 1m - else - echo "[INFO]-------------------------------------------------------------" - echo "[INFO] Congratulations, got lock to join Ignite cluster" - echo "[INFO]-------------------------------------------------------------" - break - fi - done + cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml + mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml } +# Checks status of Ignite daemon checkIgniteStatus() { proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") @@ -419,52 +109,7 @@ checkIgniteStatus() return 1 } -waitFirstIgniteNodeRegistered() -{ - echo "[INFO] Waiting for the first Ignite node to register" - - startTime=$(date +%s) - - while true; do - first_host= - - exists=$(aws s3 ls $S3_IGNITE_FIRST_NODE_LOCK_URL) - if [ -n "$exists" ]; then - rm -Rf /opt/ignite/first-node-lock - - aws s3 cp $S3_IGNITE_FIRST_NODE_LOCK_URL /opt/ignite/first-node-lock - if [ $? -ne 0 ]; then - terminate "Failed to check existing first node lock" - fi - - first_host=$(cat /opt/ignite/first-node-lock) - - rm -Rf /opt/ignite/first-node-lock - fi - - if [ -n "$first_host" ]; then - exists=$(aws s3 ls ${S3_IGNITE_NODES_DISCOVERY_URL}${first_host}) - if [ -n "$exists" ]; then - break - fi - fi - - currentTime=$(date +%s) - duration=$(( $currentTime-$startTime )) - duration=$(( $duration/60 )) - - if [ $duration -gt $NODE_STARTUP_TIME ]; then - terminate "${NODE_STARTUP_TIME}min timeout expired, but first Ignite node is still not up and running" - fi - - echo "[INFO] Waiting extra 1min" - - sleep 1m - done - - echo "[INFO] First Ignite node registered" -} - +# Gracefully starts Ignite daemon and waits until it joins Ignite cluster startIgnite() { echo "[INFO]-------------------------------------------------------------" @@ -475,13 +120,13 @@ startIgnite() setupCassandraSeeds setupIgniteSeeds - if [ "$FIRST_NODE" == "true" ]; then - aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY_URL::-1} + waitToJoinCluster + + if [ "$FIRST_NODE_LOCK" == "true" ]; then + aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} if [ $? -ne 0 ]; then - terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY_URL" + terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" fi - else - waitToJoinIgniteCluster fi proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") @@ -503,77 +148,46 @@ startIgnite() START_ATTEMPT=$(( $START_ATTEMPT+1 )) } -# Time (in minutes) to wait for Ignite/Cassandra daemon up and running -NODE_STARTUP_TIME=10 - -# Number of attempts to start (not first) Ignite daemon -NODE_START_ATTEMPTS=3 - -HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') -echo $HOST_NAME > /opt/ignite/join-lock +####################################################################################################### START_ATTEMPT=0 -FIRST_NODE="false" - +# Cleans all the previous metadata about this EC2 node unregisterNode +# Tries to get first-node lock tryToGetFirstNodeLock -if [ $? -eq 0 ]; then - FIRST_NODE="true" -fi - echo "[INFO]-----------------------------------------------------------------" -if [ "$FIRST_NODE" == "true" ]; then +if [ "$FIRST_NODE_LOCK" == "true" ]; then echo "[INFO] Starting first Ignite node" else echo "[INFO] Starting Ignite node" fi echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Ignite nodes discovery URL: $S3_IGNITE_NODES_DISCOVERY_URL" -echo "[INFO] Ignite first node lock URL: $S3_IGNITE_FIRST_NODE_LOCK_URL" -echo "[INFO] Cassandra nodes discovery URL: $S3_CASSANDRA_NODES_DISCOVERY_URL" -echo "[INFO] Start success URL: $S3_BOOTSTRAP_SUCCESS_URL" -echo "[INFO] Start failure URL: $S3_BOOTSTRAP_FAILURE_URL" -echo "[INFO] IGNITE_HOME: $IGNITE_HOME" -echo "[INFO] JAVA_HOME: $JAVA_HOME" -echo "[INFO] PATH: $PATH" +printInstanceInfo echo "[INFO]-----------------------------------------------------------------" -if [ -z "$S3_CASSANDRA_NODES_DISCOVERY_URL" ]; then - terminate "Cassandra S3 discovery URL doesn't specified" -fi - -if [[ "$S3_CASSANDRA_NODES_DISCOVERY_URL" != */ ]]; then - S3_CASSANDRA_NODES_DISCOVERY_URL=${S3_CASSANDRA_NODES_DISCOVERY_URL}/ -fi - -if [ -z "$S3_IGNITE_NODES_DISCOVERY_URL" ]; then - terminate "Ignite S3 discovery URL doesn't specified" -fi - -if [[ "$S3_IGNITE_NODES_DISCOVERY_URL" != */ ]]; then - S3_IGNITE_NODES_DISCOVERY_URL=${S3_IGNITE_NODES_DISCOVERY_URL}/ -fi - -if [ "$FIRST_NODE" != "true" ]; then - waitFirstIgniteNodeRegistered +if [ "$FIRST_NODE_LOCK" != "true" ]; then + waitFirstClusterNodeRegistered "true" else cleanupMetadata fi +# Applies Ignite environment settings from ignite-env.sh envScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/ignite-env.sh) if [ -f "$envScript" ]; then . $envScript fi +# Start Ignite daemon startIgnite startTime=$(date +%s) +# Trying multiple attempts to start Ignite daemon while true; do proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") @@ -587,9 +201,9 @@ while true; do echo $proc echo "[INFO]-----------------------------------------------------" - if [ "$FIRST_NODE" != "true" ]; then - removeClusterJoinLock - fi + # Once node joined the cluster we need to remove cluster-join lock + # to allow other EC2 nodes to acquire it and join cluster sequentially + removeClusterJoinLock break fi @@ -598,33 +212,46 @@ while true; do duration=$(( $currentTime-$startTime )) duration=$(( $duration/60 )) - if [ $duration -gt $NODE_STARTUP_TIME ]; then - if [ "$FIRST_NODE" == "true" ]; then - removeFirstNodeLock - terminate "${NODE_STARTUP_TIME}min timeout expired, but first Ignite daemon is still not up and running" + if [ $duration -gt $SERVICE_STARTUP_TIME ]; then + if [ "$FIRST_NODE_LOCK" == "true" ]; then + # If the first node of Ignite cluster failed to start Ignite daemon in SERVICE_STARTUP_TIME min, + # we will not try any other attempts and just terminate with error. Terminate function itself, will + # take care about removing all the locks holding by this node. + terminate "${SERVICE_STARTUP_TIME}min timeout expired, but first Ignite daemon is still not up and running" else + # If node isn't the first node of Ignite cluster and it failed to start we need to + # remove cluster-join lock to allow other EC2 nodes to acquire it removeClusterJoinLock - if [ $START_ATTEMPT -gt $NODE_START_ATTEMPTS ]; then - terminate "${NODE_START_ATTEMPTS} attempts exceed, but Ignite daemon is still not up and running" + # If node failed all SERVICE_START_ATTEMPTS attempts to start Ignite daemon we will not + # try anymore and terminate with error + if [ $START_ATTEMPT -gt $SERVICE_START_ATTEMPTS ]; then + terminate "${SERVICE_START_ATTEMPTS} attempts exceed, but Ignite daemon is still not up and running" fi + # New attempt to start Ignite daemon startIgnite fi continue fi + # Handling situation when Ignite daemon process abnormally terminated if [ -z "$proc" ]; then - if [ "$FIRST_NODE" == "true" ]; then - removeFirstNodeLock + # If this is the first node of Ignite cluster just terminating with error + if [ "$FIRST_NODE_LOCK" == "true" ]; then terminate "Failed to start Ignite daemon" fi + # Remove cluster-join lock to allow other EC2 nodes to acquire it removeClusterJoinLock - echo "[WARN] Failed to start Ignite daemon. Sleeping for extra 1min" - sleep 1m + + echo "[WARN] Failed to start Ignite daemon. Sleeping for extra 30sec" + sleep 30s + + # New attempt to start Ignite daemon startIgnite + continue fi @@ -632,6 +259,8 @@ while true; do sleep 30s done +# Once Ignite daemon successfully started we registering new Ignite node in S3 registerNode +# Terminating script with zero exit code terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh b/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh index 73e3c2c..1634b89 100644 --- a/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh +++ b/modules/cassandra/src/test/bootstrap/aws/logs-collector.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,86 +17,157 @@ # limitations under the License. # -echo "[INFO] Running Logs collector service" +# ----------------------------------------------------------------------------------------------- +# Logs collector daemon +# ----------------------------------------------------------------------------------------------- +# Script is launched in background by all EC2 nodes of all clusters (Cassandra, Ignite, Tests) and +# periodically (each 30 seconds) checks if specific S3 trigger file (specified by $S3_LOGS_TRIGGER_URL) +# was created or its timestamp was changed. Such an event serve as a trigger for the script +# to collect EC2 instance logs (from folder specified by $1) and upload them into specific +# S3 folder (specified by $S3_LOGS_FOLDER). +# ----------------------------------------------------------------------------------------------- + +uploadLogs() +{ + if [ ! -d "$1" ]; then + echo "[INFO] Logs directory doesn't exist: $1" + return 0 + fi -if [ -z "$1" ]; then - echo "[ERROR] Local logs directory doesn't specified" - exit 1 -fi + echo "[INFO] Uploading logs from directory: $1" -echo "[INFO] Local logs directory: $1" + dirList=$(ls $1 | head -1) -if [ -z "$2" ]; then - echo "[ERROR] S3 folder where to upload logs doesn't specified" - exit 1 -fi + if [ -z "$dirList" ]; then + echo "[INFO] Directory is empty: $1" + fi -echo "[INFO] S3 logs upload folder: $2" + for i in 0 9; + do + aws s3 sync --sse AES256 --delete "$1" "$S3_LOGS_FOLDER" + code=$? -if [ -z "$3" ]; then - echo "[ERROR] Logs collection S3 trigger URL doesn't specified" - exit 1 -fi + if [ $code -eq 0 ]; then + echo "[INFO] Successfully uploaded logs from directory: $1" + return 0 + fi -echo "[INFO] Logs collection S3 trigger URL: $3" + echo "[WARN] Failed to upload logs from $i attempt, sleeping extra 30sec" + sleep 30s + done -echo "--------------------------------------------------------------------" + echo "[ERROR] All 10 attempts to upload logs are failed for the directory: $1" +} -TRIGGER_STATE= +createNewLogsSnapshot() +{ + rm -f ~/logs-collector.snapshot.new -while true; do - sleep 1m + for log_src in "$@" + do + if [ -d "$log_src" ] || [ -f "$log_src" ]; then + ls -alR $log_src >> ~/logs-collector.snapshot.new - STATE=$(aws s3 ls $3) + fi + done +} - if [ -z "$STATE" ] || [ "$STATE" == "$TRIGGER_STATE" ]; then - continue +checkLogsChanged() +{ + createNewLogsSnapshot $@ + + if [ ! -f "~/logs-collector.snapshot" ]; then + return 1 fi - TRIGGER_STATE=$STATE + diff "~/logs-collector.snapshot" "~/logs-collector.snapshot.new" > /dev/null + + return $? +} - exists= - if [ -d "$1" ]; then - exists="true" +updateLogsSnapshot() +{ + if [ ! -f "~/logs-collector.snapshot.new" ]; then + return 0 fi - echo "[INFO] Uploading logs from $1 to $2" + rm -f "~/logs-collector.snapshot" + mv "~/logs-collector.snapshot.new" "~/logs-collector.snapshot" +} + +collectLogs() +{ + createNewLogsSnapshot + + rm -Rf ~/logs-collector-logs + mkdir -p ~/logs-collector-logs + + for log_src in "$@" + do + if [ -f "$log_src" ]; then + echo "[INFO] Collecting log file: $log_src" + cp -f $log_src ~/logs-collector-logs + elif [ -d "$log_src" ]; then + echo "[INFO] Collecting logs from folder: $log_src" + cp -Rf $log_src ~/logs-collector-logs + fi + done - if [ "$exists" != "true" ]; then - echo "[INFO] Local logs directory $1 doesn't exist, thus there is nothing to upload" - fi + uploadLogs ~/logs-collector-logs - echo "--------------------------------------------------------------------" + rm -Rf ~/logs-collector-logs - if [ "$exists" != "true" ]; then - continue - fi + updateLogsSnapshot +} - aws s3 sync --sse AES256 --delete "$1" "$2" +echo "[INFO] Running Logs collector service" - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to upload logs from $1 to $2 from first attempt" - sleep 30s +if [ -z "$1" ]; then + echo "[ERROR] Logs collection S3 trigger URL doesn't specified" + exit 1 +fi + +S3_LOGS_TRIGGER_URL=$1 + +echo "[INFO] Logs collection S3 trigger URL: $S3_LOGS_TRIGGER_URL" - aws s3 sync --sse AES256 --delete "$1" "$2" +if [ -z "$2" ]; then + echo "[ERROR] S3 folder where to upload logs doesn't specified" + exit 1 +fi - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to upload logs from $1 to $2 from second attempt" - sleep 1m +S3_LOGS_FOLDER=$2 - aws s3 sync --sse AES256 --delete "$1" "$2" +echo "[INFO] S3 logs upload folder: $S3_LOGS_FOLDER" - if [ $? -ne 0 ]; then - echo "[ERROR] Failed to upload logs from $1 to $2 from third attempt" - else - echo "[INFO] Logs successfully uploaded from $1 to $2 from third attempt" - fi - else - echo "[INFO] Logs successfully uploaded from $1 to $2 from second attempt" +shift 2 + +if [ -z "$1" ]; then + echo "[WARN] Local logs sources don't specified" +else + echo "[INFO] Local logs sources: $@" +fi + +echo "--------------------------------------------------------------------" + +TRIGGER_STATE= + +while true; do + sleep 30s + + STATE=$(aws s3 ls $S3_LOGS_TRIGGER_URL) + + if [ -z "$STATE" ] || [ "$STATE" == "$TRIGGER_STATE" ]; then + checkLogsChanged + + if [ $? -eq 0 ]; then + continue fi - else - echo "[INFO] Logs successfully uploaded from $1 to $2" fi + TRIGGER_STATE=$STATE + + collectLogs $@ /var/log/cloud-init.log /var/log/cloud-init-output.log + echo "--------------------------------------------------------------------" done http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml b/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml index 53c33a2..77ffb9e 100644 --- a/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml +++ b/modules/cassandra/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml @@ -24,7 +24,11 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd";> - <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.RoundRobinPolicy"/> + <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.TokenAwarePolicy"> + <constructor-arg type="com.datastax.driver.core.policies.LoadBalancingPolicy"> + <bean class="com.datastax.driver.core.policies.RoundRobinPolicy"/> + </constructor-arg> + </bean> <util:list id="contactPoints" value-type="java.lang.String"> ${CASSANDRA_SEEDS} @@ -114,6 +118,11 @@ <!-- Configuring persistence for "cache1" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache1"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/> @@ -128,6 +137,11 @@ <!-- Configuring persistence for "cache2" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache2"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/> @@ -142,6 +156,11 @@ <!-- Configuring persistence for "cache3" cache --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="cache3"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> <property name="readThrough" value="true"/> <property name="writeThrough" value="true"/> <property name="writeBehindEnabled" value="true"/> http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh index d00ddb6..8e6faff 100644 --- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh +++ b/modules/cassandra/src/test/bootstrap/aws/tests/tests-bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,51 +17,31 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Bootstrap script to spin up Tests cluster +# ----------------------------------------------------------------------------------------------- + +# URL to download AWS CLI tools AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip -S3_ROOT=s3://bucket/folder -S3_DOWNLOADS=$S3_ROOT/test -S3_SYSTEM=$S3_ROOT/test1 - -TESTS_PACKAGE_DONLOAD_URL=$S3_DOWNLOADS/ignite-cassandra-tests-1.6.0-SNAPSHOT.zip -TESTS_PACKAGE_ZIP=ignite-cassandra-tests-1.6.0-SNAPSHOT.zip -TESTS_PACKAGE_UNZIP_DIR=ignite-cassandra-tests - -S3_LOGS_URL=$S3_SYSTEM/logs/t-logs -S3_LOGS_TRIGGER_URL=$S3_SYSTEM/logs-trigger -S3_IGNITE_NODES_DISCOVERY_URL=$S3_SYSTEM/i-discovery -S3_CASSANDRA_NODES_DISCOVERY_URL=$S3_SYSTEM/c-discovery -S3_TEST_NODES_DISCOVERY_URL=$S3_SYSTEM/t-discovery -S3_TESTS_SUCCESS_URL=$S3_SYSTEM/t-success -S3_TESTS_FAILURE_URL=$S3_SYSTEM/t-failure -S3_TESTS_RUNNING_URL=$S3_SYSTEM/t-running -S3_TESTS_WAITING_URL=$S3_SYSTEM/t-waiting -S3_IGNITE_SUCCESS_URL=$S3_SYSTEM/i-success -S3_IGNITE_FAILURE_URL=$S3_SYSTEM/i-failure -S3_CASSANDRA_SUCCESS_URL=$S3_SYSTEM/c-success -S3_CASSANDRA_FAILURE_URL=$S3_SYSTEM/c-failure -S3_TESTS_FIRST_NODE_LOCK_URL=$S3_SYSTEM/t-first-node-lock -S3_TESTS_SUMMARY_URL=$S3_SYSTEM/t-summary.zip - -INSTANCE_REGION=us-west-2 -INSTANCE_NAME_TAG=TEST-SERVER [email protected] -INSTANCE_PROJECT_TAG=ignite - -CASSANDRA_NODES_COUNT=50 -IGNITE_NODES_COUNT=30 -TEST_NODES_COUNT=30 - -TESTS_TYPE="ignite" +# URL to download JDK +JDK_DOWNLOAD_URL=http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz + +# URL to download Ignite-Cassandra tests package - you should previously package and upload it to this place +TESTS_PACKAGE_DONLOAD_URL=s3://<bucket>/<folder>/ignite-cassandra-tests-<version>.zip +# Terminates script execution and upload logs to S3 terminate() { - if [[ "$S3_TESTS_SUCCESS_URL" != */ ]]; then - S3_TESTS_SUCCESS_URL=${S3_TESTS_SUCCESS_URL}/ + SUCCESS_URL=$S3_TESTS_SUCCESS + FAILURE_URL=$S3_TESTS_FAILURE + + if [ -n "$SUCCESS_URL" ] && [[ "$SUCCESS_URL" != */ ]]; then + SUCCESS_URL=${SUCCESS_URL}/ fi - if [[ "$S3_TESTS_FAILURE_URL" != */ ]]; then - S3_TESTS_FAILURE_URL=${S3_TESTS_FAILURE_URL}/ + if [ -n "$FAILURE_URL" ] && [[ "$FAILURE_URL" != */ ]]; then + FAILURE_URL=${FAILURE_URL}/ fi host_name=$(hostname -f | tr '[:upper:]' '[:lower:]') @@ -73,13 +53,23 @@ terminate() echo "[ERROR] Test node bootstrap failed" echo "[ERROR]-----------------------------------------------------" msg=$1 - reportFolder=${S3_TESTS_FAILURE_URL}${host_name} + + if [ -z "$FAILURE_URL" ]; then + exit 1 + fi + + reportFolder=${FAILURE_URL}${host_name} reportFile=$reportFolder/__error__ else echo "[INFO]-----------------------------------------------------" echo "[INFO] Test node bootstrap successfully completed" echo "[INFO]-----------------------------------------------------" - reportFolder=${S3_TESTS_SUCCESS_URL}${host_name} + + if [ -z "$SUCCESS_URL" ]; then + exit 0 + fi + + reportFolder=${SUCCESS_URL}${host_name} reportFile=$reportFolder/__success__ fi @@ -87,7 +77,7 @@ terminate() aws s3 rm --recursive $reportFolder if [ $? -ne 0 ]; then - echo "[ERROR] Failed drop report folder: $reportFolder" + echo "[ERROR] Failed to drop report folder: $reportFolder" fi aws s3 cp --sse AES256 /opt/bootstrap-result $reportFile @@ -104,172 +94,67 @@ terminate() exit 0 } -tagInstance() -{ - export EC2_HOME=/opt/aws/apitools/ec2 - export JAVA_HOME=/opt/jdk1.8.0_77 - export PATH=$JAVA_HOME/bin:$EC2_HOME/bin:$PATH - - INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) - if [ $? -ne 0 ]; then - terminate "Failed to get instance metadata to tag it" - fi - - if [ -n "$INSTANCE_NAME_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag Name=${INSTANCE_NAME_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: Name=${INSTANCE_NAME_TAG}" - fi - fi - - if [ -n "$INSTANCE_OWNER_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag owner=${INSTANCE_OWNER_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: owner=${INSTANCE_OWNER_TAG}" - fi - fi - - if [ -n "$INSTANCE_PROJECT_TAG" ]; then - ec2-create-tags $INSTANCE_ID --tag project=${INSTANCE_PROJECT_TAG} --region $INSTANCE_REGION - if [ $code -ne 0 ]; then - terminate "Failed to tag EC2 instance with: project=${INSTANCE_PROJECT_TAG}" - fi - fi -} - +# Downloads specified package downloadPackage() { echo "[INFO] Downloading $3 package from $1 into $2" - if [[ "$1" == s3* ]]; then - aws s3 cp $1 $2 - - if [ $? -ne 0 ]; then - echo "[WARN] Failed to download $3 package from first attempt" - rm -Rf $2 - sleep 10s - - echo "[INFO] Trying second attempt to download $3 package" + for i in 0 9; + do + if [[ "$1" == s3* ]]; then aws s3 cp $1 $2 + code=$? + else + curl "$1" -o "$2" + code=$? + fi - if [ $? -ne 0 ]; then - echo "[WARN] Failed to download $3 package from second attempt" - rm -Rf $2 - sleep 10s - - echo "[INFO] Trying third attempt to download $3 package" - aws s3 cp $1 $2 - - if [ $? -ne 0 ]; then - terminate "All three attempts to download $3 package from $1 are failed" - fi - fi + if [ $code -eq 0 ]; then + echo "[INFO] $3 package successfully downloaded from $1 into $2" + return 0 fi - else - curl "$1" -o "$2" - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - echo "[WARN] Failed to download $3 package from first attempt" - rm -Rf $2 - sleep 10s + echo "[WARN] Failed to download $3 package from $i attempt, sleeping extra 5sec" + sleep 5s + done - echo "[INFO] Trying second attempt to download $3 package" - curl "$1" -o "$2" + terminate "All 10 attempts to download $3 package from $1 are failed" +} - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - echo "[WARN] Failed to download $3 package from second attempt" - rm -Rf $2 - sleep 10s +# Downloads and setup JDK +setupJava() +{ + rm -Rf /opt/java /opt/jdk.tar.gz - echo "[INFO] Trying third attempt to download $3 package" - curl "$1" -o "$2" + echo "[INFO] Downloading 'jdk'" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "$JDK_DOWNLOAD_URL" -O /opt/jdk.tar.gz + if [ $? -ne 0 ]; then + terminate "Failed to download 'jdk'" + fi - if [ $? -ne 0 ] && [ $? -ne 6 ]; then - terminate "All three attempts to download $3 package from $1 are failed" - fi - fi - fi + echo "[INFO] Untaring 'jdk'" + tar -xvzf /opt/jdk.tar.gz -C /opt + if [ $? -ne 0 ]; then + terminate "Failed to untar 'jdk'" fi - echo "[INFO] $3 package successfully downloaded from $1 into $2" + rm -Rf /opt/jdk.tar.gz + + unzipDir=$(ls /opt | grep "jdk") + if [ "$unzipDir" != "java" ]; then + mv /opt/$unzipDir /opt/java + fi } -echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Bootstrapping Tests node" -echo "[INFO]-----------------------------------------------------------------" -echo "[INFO] Tests type: $TESTS_TYPE" -echo "[INFO] Test nodes count: $TEST_NODES_COUNT" -echo "[INFO] Ignite nodes count: $IGNITE_NODES_COUNT" -echo "[INFO] Cassandra nodes count: $CASSANDRA_NODES_COUNT" -echo "[INFO] Tests summary URL: $S3_TESTS_SUMMARY_URL" -echo "[INFO] Tests first node lock URL: $S3_TESTS_FIRST_NODE_LOCK_URL" -echo "[INFO] Logs URL: $S3_LOGS_URL" -echo "[INFO] Logs trigger URL: $S3_LOGS_TRIGGER_URL" -echo "[INFO] Tests package download URL: $TESTS_PACKAGE_DONLOAD_URL" -echo "[INFO] Test node discovery URL: $S3_TEST_NODES_DISCOVERY_URL" -echo "[INFO] Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY_URL" -echo "[INFO] Cassandra node discovery URL: $S3_CASSANDRA_NODES_DISCOVERY_URL" -echo "[INFO] Tests running URL: $S3_TESTS_RUNNING_URL" -echo "[INFO] Tests waiting URL: $S3_TESTS_WAITING_URL" -echo "[INFO] Tests success URL: $S3_TESTS_SUCCESS_URL" -echo "[INFO] Tests failure URL: $S3_TESTS_FAILURE_URL" -echo "[INFO] Ignite success URL: $S3_IGNITE_SUCCESS_URL" -echo "[INFO] Ignite failure URL: $S3_IGNITE_FAILURE_URL" -echo "[INFO] Cassandra success URL: $S3_CASSANDRA_SUCCESS_URL" -echo "[INFO] Cassandra failure URL: $S3_CASSANDRA_FAILURE_URL" -echo "[INFO]-----------------------------------------------------------------" +# Downloads and setup AWS CLI +setupAWSCLI() +{ + echo "[INFO] Installing 'awscli'" + pip install --upgrade awscli + if [ $? -eq 0 ]; then + return 0 + fi -echo "[INFO] Installing 'wget' package" -yum -y install wget -if [ $? -ne 0 ]; then - terminate "Failed to install 'wget' package" -fi - -echo "[INFO] Installing 'net-tools' package" -yum -y install net-tools -if [ $? -ne 0 ]; then - terminate "Failed to install 'net-tools' package" -fi - -echo "[INFO] Installing 'python' package" -yum -y install python -if [ $? -ne 0 ]; then - terminate "Failed to install 'python' package" -fi - -echo "[INFO] Installing 'unzip' package" -yum -y install unzip -if [ $? -ne 0 ]; then - terminate "Failed to install 'unzip' package" -fi - -rm -Rf /opt/jdk1.8.0_77 /opt/jdk-8u77-linux-x64.tar.gz - -echo "[INFO] Downloading 'jdk-8u77'" -wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz"; -O /opt/jdk-8u77-linux-x64.tar.gz -if [ $? -ne 0 ]; then - terminate "Failed to download 'jdk-8u77'" -fi - -echo "[INFO] Unzipping 'jdk-8u77'" -tar -xvzf /opt/jdk-8u77-linux-x64.tar.gz -C /opt -if [ $? -ne 0 ]; then - terminate "Failed to untar 'jdk-8u77'" -fi - -rm -Rf /opt/jdk-8u77-linux-x64.tar.gz - -downloadPackage "https://bootstrap.pypa.io/get-pip.py"; "/opt/get-pip.py" "get-pip.py" - -echo "[INFO] Installing 'pip'" -python /opt/get-pip.py -if [ $? -ne 0 ]; then - terminate "Failed to install 'pip'" -fi - -echo "[INFO] Installing 'awscli'" -pip install --upgrade awscli -if [ $? -ne 0 ]; then echo "[ERROR] Failed to install 'awscli' using pip" echo "[INFO] Trying to install awscli using zip archive" echo "[INFO] Downloading awscli zip" @@ -282,7 +167,7 @@ if [ $? -ne 0 ]; then terminate "Failed to unzip awscli zip" fi - rm -fR /opt/awscli-bundle.zip + rm -Rf /opt/awscli-bundle.zip echo "[INFO] Installing awscli" /opt/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws @@ -291,89 +176,142 @@ if [ $? -ne 0 ]; then fi echo "[INFO] Successfully installed awscli from zip archive" -fi +} -tagInstance +# Setup all the pre-requisites (packages, settings and etc.) +setupPreRequisites() +{ + echo "[INFO] Installing 'wget' package" + yum -y install wget + if [ $? -ne 0 ]; then + terminate "Failed to install 'wget' package" + fi -echo "[INFO] Creating 'ignite' group" -exists=$(cat /etc/group | grep ignite) -if [ -z "$exists" ]; then - groupadd ignite + echo "[INFO] Installing 'net-tools' package" + yum -y install net-tools if [ $? -ne 0 ]; then - terminate "Failed to create 'ignite' group" + terminate "Failed to install 'net-tools' package" fi -fi -echo "[INFO] Creating 'ignite' user" -exists=$(cat /etc/passwd | grep ignite) -if [ -z "$exists" ]; then - useradd -g ignite ignite + echo "[INFO] Installing 'python' package" + yum -y install python if [ $? -ne 0 ]; then - terminate "Failed to create 'ignite' user" + terminate "Failed to install 'python' package" fi -fi -downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/$TESTS_PACKAGE_ZIP" "Tests" + echo "[INFO] Installing 'unzip' package" + yum -y install unzip + if [ $? -ne 0 ]; then + terminate "Failed to install 'unzip' package" + fi -unzip /opt/$TESTS_PACKAGE_ZIP -d /opt -if [ $? -ne 0 ]; then - terminate "Failed to unzip tests package: $TESTS_PACKAGE_DONLOAD_URL" -fi + downloadPackage "https://bootstrap.pypa.io/get-pip.py"; "/opt/get-pip.py" "get-pip.py" -mv /opt/$TESTS_PACKAGE_UNZIP_DIR /opt/ignite-cassandra-tests + echo "[INFO] Installing 'pip'" + python /opt/get-pip.py + if [ $? -ne 0 ]; then + terminate "Failed to install 'pip'" + fi +} -if [ ! -f "/opt/ignite-cassandra-tests/cassandra-load-tests.sh" ]; then - terminate "There are no cassandra-load-tests.sh in tests package" -fi +# Downloads and setup tests package +setupTestsPackage() +{ + downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/ignite-cassandra-tests.zip" "Tests" -if [ ! -f "/opt/ignite-cassandra-tests/ignite-load-tests.sh" ]; then - terminate "There are no ignite-load-tests.sh in tests package" -fi + rm -Rf /opt/ignite-cassandra-tests -if [ ! -f "/opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template.xml" ]; then - terminate "There are no ignite-cassandra-client-template.xml in tests package" -fi + unzip /opt/ignite-cassandra-tests.zip -d /opt + if [ $? -ne 0 ]; then + terminate "Failed to unzip tests package" + fi + + rm -f /opt/ignite-cassandra-tests.zip + + unzipDir=$(ls /opt | grep "ignite-cassandra") + if [ "$unzipDir" != "ignite-cassandra-tests" ]; then + mv /opt/$unzipDir /opt/ignite-cassandra-tests + fi + + find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \; + + . /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" -if [ ! -f "/opt/$TESTS_PACKAGE_UNZIP_DIR/bootstrap/aws/logs-collector.sh" ]; then - terminate "There are no logs-collector.sh in tests package" -fi + setupNTP -chown -R ignite:ignite /opt/ignite-cassandra-tests -find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \; + echo "[INFO] Starting logs collector daemon" -cp -f /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh /opt -chown -R ignite:ignite /opt/logs-collector.sh + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh "$S3_LOGS_TRIGGER" "$S3_TESTS_LOGS/$HOST_NAME" "/opt/ignite-cassandra-tests/logs" > /opt/logs-collector.log & -#profile=/home/ignite/.bash_profile -profile=/root/.bash_profile + echo "[INFO] Logs collector daemon started: $!" -echo "export JAVA_HOME=/opt/jdk1.8.0_77" >> $profile -echo "export PATH=\$JAVA_HOME/bin:\IGNITE_HOME/bin:\$PATH" >> $profile -echo "export TESTS_TYPE=$TESTS_TYPE" >> $profile -echo "export S3_TESTS_SUMMARY_URL=$S3_TESTS_SUMMARY_URL" >> $profile -echo "export S3_CASSANDRA_NODES_DISCOVERY_URL=$S3_CASSANDRA_NODES_DISCOVERY_URL" >> $profile -echo "export S3_TEST_NODES_DISCOVERY_URL=$S3_TEST_NODES_DISCOVERY_URL" >> $profile -echo "export S3_IGNITE_NODES_DISCOVERY_URL=$S3_IGNITE_NODES_DISCOVERY_URL" >> $profile -echo "export S3_TESTS_RUNNING_URL=$S3_TESTS_RUNNING_URL" >> $profile -echo "export S3_TESTS_WAITING_URL=$S3_TESTS_WAITING_URL" >> $profile -echo "export S3_TESTS_SUCCESS_URL=$S3_TESTS_SUCCESS_URL" >> $profile -echo "export S3_TESTS_FAILURE_URL=$S3_TESTS_FAILURE_URL" >> $profile -echo "export S3_IGNITE_SUCCESS_URL=$S3_IGNITE_SUCCESS_URL" >> $profile -echo "export S3_IGNITE_FAILURE_URL=$S3_IGNITE_FAILURE_URL" >> $profile -echo "export S3_CASSANDRA_SUCCESS_URL=$S3_CASSANDRA_SUCCESS_URL" >> $profile -echo "export S3_CASSANDRA_FAILURE_URL=$S3_CASSANDRA_FAILURE_URL" >> $profile -echo "export S3_TESTS_FIRST_NODE_LOCK_URL=$S3_TESTS_FIRST_NODE_LOCK_URL" >> $profile -echo "export CASSANDRA_NODES_COUNT=$CASSANDRA_NODES_COUNT" >> $profile -echo "export IGNITE_NODES_COUNT=$IGNITE_NODES_COUNT" >> $profile -echo "export TEST_NODES_COUNT=$TEST_NODES_COUNT" >> $profile -echo "export S3_LOGS_TRIGGER_URL=$S3_LOGS_TRIGGER_URL" >> $profile + echo "----------------------------------------------------------------------------------------" + printInstanceInfo + echo "----------------------------------------------------------------------------------------" + tagInstance + bootstrapGangliaAgent "test" 8643 + + ################################################### + # Extra configuration specific only for test node # + ################################################### + + echo "[INFO] Installing bc package" + + yum -y install bc + + if [ $? -ne 0 ]; then + terminate "Failed to install bc package" + fi + + echo "[INFO] Installing zip package" + + yum -y install zip + + if [ $? -ne 0 ]; then + terminate "Failed to install zip package" + fi + + echo "[INFO] Creating 'ignite' group" + exists=$(cat /etc/group | grep ignite) + if [ -z "$exists" ]; then + groupadd ignite + if [ $? -ne 0 ]; then + terminate "Failed to create 'ignite' group" + fi + fi + + echo "[INFO] Creating 'ignite' user" + exists=$(cat /etc/passwd | grep ignite) + if [ -z "$exists" ]; then + useradd -g ignite ignite + if [ $? -ne 0 ]; then + terminate "Failed to create 'ignite' user" + fi + fi + + mkdir -p /opt/ignite-cassandra-tests/logs + chown -R ignite:ignite /opt/ignite-cassandra-tests + + echo "export JAVA_HOME=/opt/java" >> $1 + echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> $1 +} + +################################################################################################################### + +echo "[INFO]-----------------------------------------------------------------" +echo "[INFO] Bootstrapping Tests node" +echo "[INFO]-----------------------------------------------------------------" -HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') +setupPreRequisites +setupJava +setupAWSCLI +setupTestsPackage "/root/.bash_profile" -/opt/logs-collector.sh "/opt/ignite-cassandra-tests/logs" "$S3_LOGS_URL/$HOST_NAME" "$S3_LOGS_TRIGGER_URL" > /opt/ignite-cassandra-tests/logs-collector.log & +cmd="/opt/ignite-cassandra-tests/bootstrap/aws/tests/tests-manager.sh" -cmd="/opt/ignite-cassandra-tests/bootstrap/aws/tests/tests-run.sh" +#sudo -u ignite -g ignite sh -c "$cmd > /opt/ignite-cassandra-tests/tests-manager" & -#sudo -u ignite -g ignite sh -c "$cmd | tee /opt/ignite-cassandra-tests/start.log" +$cmd > /opt/ignite-cassandra-tests/logs/tests-manager.log & -$cmd | tee /opt/ignite-cassandra-tests/start.log \ No newline at end of file +terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh new file mode 100644 index 0000000..c0f5d6b --- /dev/null +++ b/modules/cassandra/src/test/bootstrap/aws/tests/tests-manager.sh @@ -0,0 +1,458 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Tests manager daemon +# ----------------------------------------------------------------------------------------------- +# Script is launched in background by all nodes of Tests cluster and +# periodically (each 30 seconds) checks if specific S3 trigger file was created or +# its timestamp was changed. Such an event serve as a trigger for the script to start +# preparing to run load tests. +# ----------------------------------------------------------------------------------------------- + +#profile=/home/ignite/.bash_profile +profile=/root/.bash_profile + +. $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" + +# Switch test node to IDLE state +switchToIdleState() +{ + if [ "$NODE_STATE" != "IDLE" ]; then + echo "[INFO] Switching node to IDLE state" + dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_IDLE" + NODE_STATE="IDLE" + echo "[INFO] Node was switched to IDLE state" + fi +} + +# Switch test node to PREPARING state +switchToPreparingState() +{ + if [ "$NODE_STATE" != "PREPARING" ]; then + echo "[INFO] Switching node to PREPARING state" + dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_IDLE" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_PREPARING" + NODE_STATE="PREPARING" + echo "[INFO] Node was switched to PREPARING state" + fi +} + +# Switch test node to WAITING state +switchToWaitingState() +{ + if [ "$NODE_STATE" != "WAITING" ]; then + echo "[INFO] Switching node to WAITING state" + dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_WAITING" + NODE_STATE="WAITING" + echo "[INFO] Node was switched to WAITING state" + fi +} + +# Switch test node to RUNNING state +switchToRunningState() +{ + if [ "$NODE_STATE" != "RUNNING" ]; then + echo "[INFO] Switching node to RUNNING state" + dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_WAITING" + createStateFlag "$S3_TESTS_RUNNING" + NODE_STATE="RUNNING" + echo "[INFO] Node was switched to RUNNING state" + fi +} + +# Creates appropriate state flag for the node in S3 +createStateFlag() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + aws s3 cp --sse AES256 /etc/hosts ${1}${HOST_NAME} + if [ $? -ne 0 ]; then + terminate "Failed to create state flag: ${1}${HOST_NAME}" + fi +} + +# Drops appropriate state flag for the node in S3 +dropStateFlag() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + for flagUrl in "$@" + do + exists=$(aws s3 ls ${flagUrl}${HOST_NAME}) + if [ -n "$exists" ]; then + aws s3 rm ${flagUrl}${HOST_NAME} + if [ $? -ne 0 ]; then + terminate "Failed to drop state flag: ${flagUrl}${HOST_NAME}" + fi + fi + done +} + +# Removes tests summary report from S3 +dropTestsSummary() +{ + exists=$(aws s3 ls $S3_TESTS_SUMMARY) + if [ -z "$exists" ]; then + return 0 + fi + + aws s3 rm $S3_TESTS_SUMMARY + if [ $? -ne 0 ]; then + terminate "Failed to drop tests summary info: $S3_TESTS_SUMMARY" + fi +} + +# Recreate all the necessary Cassandra artifacts before running Load tests +recreateCassandraArtifacts() +{ + /opt/ignite-cassandra-tests/recreate-cassandra-artifacts.sh + if [ $? -ne 0 ]; then + terminate "Failed to recreate Cassandra artifacts" + fi +} + +# Setups Cassandra seeds for this Tests node being able to connect to Cassandra. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. +setupCassandraSeeds() +{ + if [ $CASSANDRA_NODES_COUNT -eq 0 ]; then + return 0 + fi + + setupClusterSeeds "cassandra" + + CASSANDRA_SEEDS1=$(echo $CLUSTER_SEEDS | sed -r "s/ /,/g") + CASSANDRA_SEEDS2= + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + CASSANDRA_SEEDS2="${CASSANDRA_SEEDS2}<value>$seed<\/value>" + done + + echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS1" + + echo "contact.points=$CASSANDRA_SEEDS1" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/connection.properties + + cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS2/g" > /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml +} + +# Setups Ignite nodes for this Tests node being able to connect to Ignite. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. +setupIgniteSeeds() +{ + if [ $IGNITE_NODES_COUNT -eq 0 ]; then + return 0 + fi + + setupClusterSeeds "ignite" + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + IGNITE_SEEDS= + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" + done + + echo "[INFO] Using Ignite seeds: $IGNITE_SEEDS" + + cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/persistence/primitive/ignite-remote-client-config.xml + rm -f /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml +} + +# Setups Cassandra credentials to connect to Cassandra cluster +setupCassandraCredentials() +{ + echo "admin.user=cassandra" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "admin.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "regular.user=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "regular.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties +} + +# Triggering first time tests execution for all nodes in the Tests cluster +triggerFirstTimeTestsExecution() +{ + if [ -z "$TESTS_TYPE" ]; then + return 0 + fi + + tryToGetFirstNodeLock + if [ $? -ne 0 ]; then + return 0 + fi + + sleep 30s + + echo "[INFO] Triggering first time tests execution" + + echo "TESTS_TYPE=$TESTS_TYPE" > /opt/ignite-cassandra-tests/tests-trigger + echo "#--------------------------------------------------" >> /opt/ignite-cassandra-tests/tests-trigger + echo "" >> /opt/ignite-cassandra-tests/tests-trigger + cat /opt/ignite-cassandra-tests/settings/tests.properties >> /opt/ignite-cassandra-tests/tests-trigger + + aws s3 cp --sse AES256 /opt/ignite-cassandra-tests/tests-trigger $S3_TESTS_TRIGGER + code=$? + + rm -f /opt/ignite-cassandra-tests/tests-trigger + + if [ $code -ne 0 ]; then + terminate "Failed to create tests trigger: $S3_TESTS_TRIGGER" + fi +} + +# Cleans previously created logs from S3 +cleanPreviousLogs() +{ + for logFile in /opt/ignite-cassandra-tests/logs/* + do + managerLog=$(echo $logFile | grep "tests-manager") + if [ -z "$managerLog" ]; then + rm -Rf $logFile + fi + done + + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + aws s3 rm --recursive ${S3_TESTS_FAILURE}${HOST_NAME} + aws s3 rm --recursive ${S3_TESTS_SUCCESS}${HOST_NAME} +} + +# Uploads tests logs to S3 +uploadTestsLogs() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + if [ -f "/opt/ignite-cassandra-tests/logs/__success__" ]; then + logsFolder=${S3_TESTS_SUCCESS}${HOST_NAME} + else + logsFolder=${S3_TESTS_FAILURE}${HOST_NAME} + fi + + aws s3 rm --recursive $logsFolder + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to drop logs folder: $logsFolder" + fi + + if [ -d "/opt/ignite-cassandra-tests/logs" ]; then + aws s3 sync --sse AES256 /opt/ignite-cassandra-tests/logs $logsFolder + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to export tests logs to: $logsFolder" + fi + fi +} + +# Runs tests-report.sh to prepare tests summary report +buildTestsSummaryReport() +{ + reportScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/tests-report.sh) + $reportScript + + if [ -n "$S3_LOGS_TRIGGER" ]; then + aws s3 cp --sse AES256 /etc/hosts $S3_LOGS_TRIGGER + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to trigger logs collection" + fi + fi +} + +# Running load tests +runLoadTests() +{ + cd /opt/ignite-cassandra-tests + + if [ "$TESTS_TYPE" == "ignite" ]; then + echo "[INFO] Running Ignite load tests" + ./ignite-load-tests.sh & + else + echo "[INFO] Running Cassandra load tests" + ./cassandra-load-tests.sh & + fi + + testsJobId=$! + + echo "[INFO] Tests job id: $testsJobId" + + sleep 1m + + LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) + LOGS_SNAPSHOT_TIME=$(date +%s) + + TERMINATED= + + # tests monitoring + while true; do + proc=$(ps -ef | grep java | grep "org.apache.ignite.tests") + if [ -z "$proc" ]; then + break + fi + + NEW_LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) + NEW_LOGS_SNAPSHOT_TIME=$(date +%s) + + # if logs state updated it means that tests are running and not stuck + if [ "$LOGS_SNAPSHOT" != "$NEW_LOGS_SNAPSHOT" ]; then + LOGS_SNAPSHOT=$NEW_LOGS_SNAPSHOT + LOGS_SNAPSHOT_TIME=$NEW_LOGS_SNAPSHOT_TIME + continue + fi + + duration=$(( $NEW_LOGS_SNAPSHOT_TIME-$LOGS_SNAPSHOT_TIME )) + duration=$(( $duration/60 )) + + # if logs wasn't updated during 5min it means that load tests stuck + if [ $duration -gt 5 ]; then + proc=($proc) + kill -9 ${proc[1]} + TERMINATED="true" + break + fi + + echo "[INFO] Waiting extra 30sec for load tests to complete" + + sleep 30s + done + + rm -f /opt/ignite-cassandra-tests/logs/tests.properties + cp /opt/ignite-cassandra-tests/settings/tests.properties /opt/ignite-cassandra-tests/logs + + if [ "$TERMINATED" == "true" ]; then + echo "[ERROR] Load tests stuck, tests process terminated" + echo "Load tests stuck, tests process terminated" > /opt/ignite-cassandra-tests/logs/__error__ + return 0 + fi + + failed= + if [ "$TESTS_TYPE" == "cassandra" ]; then + failed=$(cat /opt/ignite-cassandra-tests/cassandra-load-tests.log | grep "load tests execution failed") + else + failed=$(cat /opt/ignite-cassandra-tests/ignite-load-tests.log | grep "load tests execution failed") + fi + + if [ -n "$failed" ]; then + echo "[ERROR] Load tests execution failed" + echo "Load tests execution failed" > /opt/ignite-cassandra-tests/logs/__error__ + else + echo "[INFO] Load tests execution successfully completed" + echo "Load tests execution successfully completed" > /opt/ignite-cassandra-tests/logs/__success__ + fi +} + +####################################################################################################### + +sleep 1m + +NODE_STATE= +TRIGGER_STATE= + +printInstanceInfo +setupCassandraCredentials +switchToIdleState + +triggerFirstTimeTestsExecution + +registerNode + +while true; do + # switching state to IDLE + switchToIdleState + + sleep 30s + + NEW_TRIGGER_STATE=$(aws s3 ls $S3_TESTS_TRIGGER | xargs) + if [ -z "$NEW_TRIGGER_STATE" ] || [ "$NEW_TRIGGER_STATE" == "$TRIGGER_STATE" ]; then + continue + fi + + echo "----------------------------------------------------------------------" + echo "[INFO] Tests trigger changed" + echo "----------------------------------------------------------------------" + echo "[INFO] Old trigger: $TRIGGER_STATE" + echo "----------------------------------------------------------------------" + echo "[INFO] New trigger: $NEW_TRIGGER_STATE" + echo "----------------------------------------------------------------------" + + TRIGGER_STATE=$NEW_TRIGGER_STATE + + aws s3 cp $S3_TESTS_TRIGGER /opt/ignite-cassandra-tests/tests-trigger + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to download tests trigger info from: $S3_TESTS_TRIGGER" + continue + fi + + TESTS_TYPE=$(cat /opt/ignite-cassandra-tests/tests-trigger | grep TESTS_TYPE | xargs | sed -r "s/TESTS_TYPE=//g") + if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then + rm -f /opt/ignite-cassandra-tests/tests-trigger + echo "[ERROR] Incorrect tests type specified in the trigger info: $S3_TESTS_TRIGGER" + continue + fi + + rm -f /opt/ignite-cassandra-tests/settings/tests.properties + mv -f /opt/ignite-cassandra-tests/tests-trigger /opt/ignite-cassandra-tests/settings/tests.properties + + waitAllTestNodesCompletedTests + + # switching state to PREPARING + switchToPreparingState + + waitAllClusterNodesReady "cassandra" + waitAllClusterNodesReady "ignite" + setupCassandraSeeds + setupIgniteSeeds + + cleanPreviousLogs + + tryToGetFirstNodeLock + if [ $? -eq 0 ]; then + dropTestsSummary + recreateCassandraArtifacts + fi + + # switching state to WAITING + switchToWaitingState + + waitAllClusterNodesReady "test" + + if [ "$FIRST_NODE_LOCK" == "true" ]; then + aws s3 rm $S3_TESTS_TRIGGER + fi + + # switching state to RUNNING + switchToRunningState + + runLoadTests + uploadTestsLogs + + tryToGetFirstNodeLock + if [ $? -eq 0 ]; then + waitAllTestNodesCompletedTests + buildTestsSummaryReport + removeFirstNodeLock + fi +done \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/ee8ac73f/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh b/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh index 762dc6f..1576d57 100644 --- a/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh +++ b/modules/cassandra/src/test/bootstrap/aws/tests/tests-report.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,118 +17,20 @@ # limitations under the License. # +# ----------------------------------------------------------------------------------------------- +# Tests report builder +# ----------------------------------------------------------------------------------------------- +# Script is used to analyze load tests logs collected from all 'Tests' cluster nodes and build +# summary report +# ----------------------------------------------------------------------------------------------- + #profile=/home/ignite/.bash_profile profile=/root/.bash_profile . $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" -validate() -{ - if [ -z "$TESTS_TYPE" ]; then - terminate "Tests type 'ignite' or 'cassandra' should be specified" - fi - - if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then - terminate "Incorrect tests type specified: $TESTS_TYPE" - fi - - if [ -z "$S3_TESTS_SUCCESS_URL" ]; then - terminate "Tests success URL doesn't specified" - fi - - if [[ "$S3_TESTS_SUCCESS_URL" != */ ]]; then - S3_TESTS_SUCCESS_URL=${S3_TESTS_SUCCESS_URL}/ - fi - - if [ -z "$S3_TESTS_FAILURE_URL" ]; then - terminate "Tests failure URL doesn't specified" - fi - - if [[ "$S3_TESTS_FAILURE_URL" != */ ]]; then - S3_TESTS_FAILURE_URL=${S3_TESTS_FAILURE_URL}/ - fi - - if [ -z "$S3_TESTS_RUNNING_URL" ]; then - terminate "Tests running URL doesn't specified" - fi - - if [[ "$S3_TESTS_RUNNING_URL" != */ ]]; then - S3_TESTS_RUNNING_URL=${S3_TESTS_RUNNING_URL}/ - fi - - if [ -z "$S3_TESTS_WAITING_URL" ]; then - terminate "Tests waiting URL doesn't specified" - fi - - if [[ "$S3_TESTS_WAITING_URL" != */ ]]; then - S3_TESTS_WAITING_URL=${S3_TESTS_WAITING_URL}/ - fi - - if [ -z "$S3_IGNITE_SUCCESS_URL" ]; then - terminate "Ignite success URL doesn't specified" - fi - - if [[ "$S3_IGNITE_SUCCESS_URL" != */ ]]; then - S3_IGNITE_SUCCESS_URL=${S3_IGNITE_SUCCESS_URL}/ - fi - - if [ -z "$S3_IGNITE_FAILURE_URL" ]; then - terminate "Ignite failure URL doesn't specified" - fi - - if [[ "$S3_IGNITE_FAILURE_URL" != */ ]]; then - S3_IGNITE_FAILURE_URL=${S3_IGNITE_FAILURE_URL}/ - fi - - if [ -z "$S3_CASSANDRA_SUCCESS_URL" ]; then - terminate "Cassandra success URL doesn't specified" - fi - - if [[ "$S3_CASSANDRA_SUCCESS_URL" != */ ]]; then - S3_CASSANDRA_SUCCESS_URL=${S3_CASSANDRA_SUCCESS_URL}/ - fi - - if [ -z "$S3_CASSANDRA_FAILURE_URL" ]; then - terminate "Cassandra failure URL doesn't specified" - fi - - if [[ "$S3_CASSANDRA_FAILURE_URL" != */ ]]; then - S3_CASSANDRA_FAILURE_URL=${S3_CASSANDRA_FAILURE_URL}/ - fi - - if [ -z "$S3_TEST_NODES_DISCOVERY_URL" ]; then - terminate "Tests S3 discovery URL doesn't specified" - fi - - if [[ "$S3_TEST_NODES_DISCOVERY_URL" != */ ]]; then - S3_TEST_NODES_DISCOVERY_URL=${S3_TEST_NODES_DISCOVERY_URL}/ - fi - - if [ -z "$S3_CASSANDRA_NODES_DISCOVERY_URL" ]; then - terminate "Cassandra S3 discovery URL doesn't specified" - fi - - if [[ "$S3_CASSANDRA_NODES_DISCOVERY_URL" != */ ]]; then - S3_CASSANDRA_NODES_DISCOVERY_URL=${S3_CASSANDRA_NODES_DISCOVERY_URL}/ - fi - - if [ -z "$S3_IGNITE_NODES_DISCOVERY_URL" ]; then - terminate "Ignite S3 discovery URL doesn't specified" - fi - - if [[ "$S3_CASSANDRA_NODES_DISCOVERY_URL" != */ ]]; then - S3_CASSANDRA_NODES_DISCOVERY_URL=${S3_CASSANDRA_NODES_DISCOVERY_URL}/ - fi - - if [ -z "$S3_IGNITE_NODES_DISCOVERY_URL" ]; then - terminate "Ignite S3 discovery URL doesn't specified" - fi - - if [[ "$S3_IGNITE_NODES_DISCOVERY_URL" != */ ]]; then - S3_IGNITE_NODES_DISCOVERY_URL=${S3_IGNITE_NODES_DISCOVERY_URL}/ - fi -} - +# Building tests summary report reportTestsSummary() { echo "[INFO] Preparing tests results summary" @@ -145,8 +47,8 @@ reportTestsSummary() mkdir -p $SUCCEED_NODES_DIR mkdir -p $FAILED_NODES_DIR - aws s3 ls $S3_TESTS_SUCCESS_URL | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $SUCCEED_NODES_FILE - aws s3 ls $S3_TESTS_FAILURE_URL | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $FAILED_NODES_FILE + aws s3 ls $S3_TESTS_SUCCESS | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $SUCCEED_NODES_FILE + aws s3 ls $S3_TESTS_FAILURE | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $FAILED_NODES_FILE succeedCount=$(cat $SUCCEED_NODES_FILE | wc -l) failedCount=$(cat $FAILED_NODES_FILE | wc -l) @@ -164,7 +66,7 @@ reportTestsSummary() cat $SUCCEED_NODES_FILE >> $REPORT_FILE echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE - aws s3 sync --delete $S3_TESTS_SUCCESS_URL $SUCCEED_NODES_DIR + aws s3 sync --delete $S3_TESTS_SUCCESS $SUCCEED_NODES_DIR if [ $? -ne 0 ]; then echo "[ERROR] Failed to get succeed tests details" else @@ -178,7 +80,7 @@ reportTestsSummary() cat $FAILED_NODES_FILE >> $REPORT_FILE echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE - aws sync --delete $S3_TESTS_FAILURE_URL $FAILED_NODES_DIR + aws sync --delete $S3_TESTS_FAILURE $FAILED_NODES_DIR if [ $? -ne 0 ]; then echo "[ERROR] Failed to get failed tests details" else @@ -204,34 +106,35 @@ reportTestsSummary() return 1 fi - aws s3 cp --sse AES256 $HOME/tests-summary.zip $S3_TESTS_SUMMARY_URL + aws s3 cp --sse AES256 $HOME/tests-summary.zip $S3_TESTS_SUMMARY if [ $? -ne 0 ]; then echo "-------------------------------------------------------------------------------------" - echo "[ERROR] Failed to uploat tests summary archive to: $S3_TESTS_SUMMARY_URL" + echo "[ERROR] Failed to uploat tests summary archive to: $S3_TESTS_SUMMARY" echo "-------------------------------------------------------------------------------------" else echo "-------------------------------------------------------------------------------------" - echo "[INFO] Tests results summary uploaded to: $S3_TESTS_SUMMARY_URL" + echo "[INFO] Tests results summary uploaded to: $S3_TESTS_SUMMARY" echo "-------------------------------------------------------------------------------------" fi rm -f $HOME/tests-summary.zip } +# Creates report for succeed tests reportSucceedTestsStatistics() { - writeMsg=0 - writeErrors=0 - writeSpeed=0 - blkWriteMsg=0 - blkWriteErrors=0 - blkWriteSpeed=0 - readMsg=0 - readErrors=0 - readSpeed=0 - blkReadMsg=0 - blkReadErrors=0 - blkReadSpeed=0 + writeMsg="0" + writeErrors="0" + writeSpeed="0" + blkWriteMsg="0" + blkWriteErrors="0" + blkWriteSpeed="0" + readMsg="0" + readErrors="0" + readSpeed="0" + blkReadMsg="0" + blkReadErrors="0" + blkReadSpeed="0" writeErrNodes= blkWriteErrNodes= @@ -271,7 +174,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^WRITE messages" | sed -r "s/WRITE messages: //g" | xargs) if [ -n "$cnt" ]; then - writeMsg=$(( $writeMsg+$cnt )) + writeMsg=$(bc <<< "$writeMsg + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] WRITE messages: $cnt" else @@ -292,7 +195,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^WRITE errors" | sed -r "s/WRITE errors: //g" | sed -r "s/,.*//g" | xargs) if [ -n "$cnt" ]; then echo "[INFO] WRITE errors: $cnt" - writeErrors=$(( $writeErrors+$cnt )) + writeErrors=$(bc <<< "$writeErrors + $cnt") if [ $cnt -ne 0 ]; then if [ -n "$writeErrNodes" ]; then writeErrNodes="${writeErrNodes}, " @@ -309,7 +212,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^WRITE speed" | sed -r "s/WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) if [ -n "$cnt" ]; then - writeSpeed=$(( $writeSpeed+$cnt )) + writeSpeed=$(bc <<< "$writeSpeed + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] WRITE speed: $cnt msg/sec" else @@ -329,7 +232,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_WRITE messages" | sed -r "s/BULK_WRITE messages: //g" | xargs) if [ -n "$cnt" ]; then - blkWriteMsg=$(( $blkWriteMsg+$cnt )) + blkWriteMsg=$(bc <<< "$blkWriteMsg + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] BULK_WRITE messages: $cnt" else @@ -349,7 +252,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_WRITE errors" | sed -r "s/BULK_WRITE errors: //g" | sed -r "s/,.*//g" | xargs) if [ -n "$cnt" ]; then - blkWriteErrors=$(( $blkWriteErrors+$cnt )) + blkWriteErrors=$(bc <<< "$blkWriteErrors + $cnt") echo "[INFO] BULK_WRITE errors: $cnt" if [ $cnt -ne 0 ]; then if [ -n "$blkWriteErrNodes" ]; then @@ -367,7 +270,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_WRITE speed" | sed -r "s/BULK_WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) if [ -n "$cnt" ]; then - blkWriteSpeed=$(( $blkWriteSpeed+$cnt )) + blkWriteSpeed=$(bc <<< "$blkWriteSpeed + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] BULK_WRITE speed: $cnt msg/sec" else @@ -387,7 +290,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^READ messages" | sed -r "s/READ messages: //g" | xargs) if [ -n "$cnt" ]; then - readMsg=$(( $readMsg+$cnt )) + readMsg=$(bc <<< "$readMsg + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] READ messages: $cnt" else @@ -407,7 +310,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^READ errors" | sed -r "s/READ errors: //g" | sed -r "s/,.*//g" | xargs) if [ -n "$cnt" ]; then - readErrors=$(( $readErrors+$cnt )) + readErrors=$(bc <<< "$readErrors + $cnt") echo "[INFO] READ errors: $cnt" if [ $cnt -ne 0 ]; then if [ -n "$readErrNodes" ]; then @@ -425,7 +328,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^READ speed" | sed -r "s/READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) if [ -n "$cnt" ]; then - readSpeed=$(( $readSpeed+$cnt )) + readSpeed=$(bc <<< "$readSpeed + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] READ speed: $cnt msg/sec" else @@ -445,7 +348,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_READ messages" | sed -r "s/BULK_READ messages: //g" | xargs) if [ -n "$cnt" ]; then - blkReadMsg=$(( $blkReadMsg+$cnt )) + blkReadMsg=$(bc <<< "$blkReadMsg + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] BULK_READ messages: $cnt" else @@ -465,7 +368,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_READ errors" | sed -r "s/BULK_READ errors: //g" | sed -r "s/,.*//g" | xargs) if [ -n "$cnt" ]; then - blkReadErrors=$(( $blkReadErrors+$cnt )) + blkReadErrors=$(bc <<< "$blkReadErrors + $cnt") echo "[INFO] BULK_READ errors: $cnt" if [ $cnt -ne 0 ]; then if [ -n "$blkReadErrNodes" ]; then @@ -483,7 +386,7 @@ reportSucceedTestsStatistics() cnt=$(cat $logFile | grep "^BULK_READ speed" | sed -r "s/BULK_READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) if [ -n "$cnt" ]; then - blkReadSpeed=$(( $blkReadSpeed+$cnt )) + blkReadSpeed=$(bc <<< "$blkReadSpeed + $cnt") if [ $cnt -ne 0 ]; then echo "[INFO] BULK_READ speed: $cnt msg/sec" else @@ -565,6 +468,7 @@ reportSucceedTestsStatistics() rm -f $tmpFile } +# Creates report for failed tests reportFailedTestsDetailes() { for dir in $2/* @@ -586,5 +490,10 @@ reportFailedTestsDetailes() done } -validate +####################################################################################################### + +if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then + terminate "Incorrect tests type specified: $TESTS_TYPE" +fi + reportTestsSummary \ No newline at end of file
