http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-env.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-env.sh b/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-env.sh new file mode 100644 index 0000000..bfe3371 --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-env.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Environment setup script from Ignite +# ----------------------------------------------------------------------------------------------- + +JVM_OPTS="-Xms10g -Xmx10g -server -XX:+AggressiveOpts -XX:MaxMetaspaceSize=256m" +JVM_OPTS="$JVM_OPTS -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseTLAB -XX:NewSize=128m -XX:MaxNewSize=768m" +#JVM_OPTS="$JVM_OPTS -XX:MaxTenuringThreshold=0 -XX:SurvivorRatio=1024 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60" +JVM_OPTS="$JVM_OPTS -Xss16m" + +export JVM_OPTS
http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-start.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-start.sh b/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-start.sh new file mode 100644 index 0000000..f2c1557 --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/ignite/ignite-start.sh @@ -0,0 +1,266 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Script to start Ignite daemon (used by ignite-bootstrap.sh) +# ----------------------------------------------------------------------------------------------- + +#profile=/home/ignite/.bash_profile +profile=/root/.bash_profile + +. $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "ignite" + +# Setups Cassandra seeds for this Ignite node being able to connect to Cassandra. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. +setupCassandraSeeds() +{ + setupClusterSeeds "cassandra" "true" + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + CASSANDRA_SEEDS= + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + CASSANDRA_SEEDS="${CASSANDRA_SEEDS}<value>$seed<\/value>" + done + + cat /opt/ignite/config/ignite-cassandra-server-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server.xml +} + +# Setups Ignite nodes which this EC2 Ignite node will use to send its metadata and join Ignite cluster +setupIgniteSeeds() +{ + if [ "$FIRST_NODE_LOCK" == "true" ]; then + echo "[INFO] Setting up Ignite seeds" + + CLUSTER_SEEDS="127.0.0.1:47500..47509" + + echo "[INFO] Using localhost address as a seed for the first Ignite node: $CLUSTER_SEEDS" + + aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} + if [ $? -ne 0 ]; then + terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" + fi + else + setupClusterSeeds "ignite" "true" + fi + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + IGNITE_SEEDS= + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" + done + + cat /opt/ignite/config/ignite-cassandra-server.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite/config/ignite-cassandra-server1.xml + mv -f /opt/ignite/config/ignite-cassandra-server1.xml /opt/ignite/config/ignite-cassandra-server.xml +} + +# Checks status of Ignite daemon +checkIgniteStatus() +{ + proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") + + nodeId= + nodeAddrs= + nodePorts= + topology= + metrics= + + logFile=$(ls /opt/ignite/work/log/ | grep "\.log$") + if [ -n "$logFile" ]; then + logFile=/opt/ignite/work/log/$logFile + nodeId=$(cat $logFile | grep "Local node \[ID") + nodeAddrs=$(cat $logFile | grep "Local node addresses:") + nodePorts=$(cat $logFile | grep "Local ports:") + topology=$(cat $logFile | grep "Topology snapshot") + metrics=$(cat $logFile | grep "Metrics for local node" | head -n 1) + fi + + if [ -n "$nodeId" ] && [ -n "$nodeAddrs" ] && [ -n "$nodePorts" ] && [ -n "$topology" ] && [ -n "$metrics" ] && [ -n "$proc" ]; then + sleep 30s + return 0 + fi + + return 1 +} + +# Gracefully starts Ignite daemon and waits until it joins Ignite cluster +startIgnite() +{ + echo "[INFO]-------------------------------------------------------------" + echo "[INFO] Trying attempt $START_ATTEMPT to start Ignite daemon" + echo "[INFO]-------------------------------------------------------------" + echo "" + + setupCassandraSeeds + setupIgniteSeeds + + waitToJoinCluster + + if [ "$FIRST_NODE_LOCK" == "true" ]; then + aws s3 rm --recursive ${S3_IGNITE_NODES_DISCOVERY::-1} + if [ $? -ne 0 ]; then + terminate "Failed to clean Ignite node discovery URL: $S3_IGNITE_NODES_DISCOVERY" + fi + fi + + proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") + proc=($proc) + + if [ -n "${proc[1]}" ]; then + echo "[INFO] Terminating existing Ignite process ${proc[1]}" + kill -9 ${proc[1]} + fi + + echo "[INFO] Starting Ignite" + rm -Rf /opt/ignite/work/* + /opt/ignite/bin/ignite.sh /opt/ignite/config/ignite-cassandra-server.xml & + + echo "[INFO] Ignite job id: $!" + + sleep 1m + + START_ATTEMPT=$(( $START_ATTEMPT+1 )) +} + +####################################################################################################### + +START_ATTEMPT=0 + +# Cleans all the previous metadata about this EC2 node +unregisterNode + +# Tries to get first-node lock +tryToGetFirstNodeLock + +echo "[INFO]-----------------------------------------------------------------" + +if [ "$FIRST_NODE_LOCK" == "true" ]; then + echo "[INFO] Starting first Ignite node" +else + echo "[INFO] Starting Ignite node" +fi + +echo "[INFO]-----------------------------------------------------------------" +printInstanceInfo +echo "[INFO]-----------------------------------------------------------------" + +if [ "$FIRST_NODE_LOCK" != "true" ]; then + waitFirstClusterNodeRegistered "true" +else + cleanupMetadata +fi + +# Applies Ignite environment settings from ignite-env.sh +envScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/ignite-env.sh) +if [ -f "$envScript" ]; then + . $envScript +fi + +# Start Ignite daemon +startIgnite + +startTime=$(date +%s) + +# Trying multiple attempts to start Ignite daemon +while true; do + proc=$(ps -ef | grep java | grep "org.apache.ignite.startup.cmdline.CommandLineStartup") + + checkIgniteStatus + + if [ $? -eq 0 ]; then + sleep 1m + echo "[INFO]-----------------------------------------------------" + echo "[INFO] Ignite daemon successfully started" + echo "[INFO]-----------------------------------------------------" + echo $proc + echo "[INFO]-----------------------------------------------------" + + # Once node joined the cluster we need to remove cluster-join lock + # to allow other EC2 nodes to acquire it and join cluster sequentially + removeClusterJoinLock + + break + fi + + currentTime=$(date +%s) + duration=$(( $currentTime-$startTime )) + duration=$(( $duration/60 )) + + if [ $duration -gt $SERVICE_STARTUP_TIME ]; then + if [ "$FIRST_NODE_LOCK" == "true" ]; then + # If the first node of Ignite cluster failed to start Ignite daemon in SERVICE_STARTUP_TIME min, + # we will not try any other attempts and just terminate with error. Terminate function itself, will + # take care about removing all the locks holding by this node. + terminate "${SERVICE_STARTUP_TIME}min timeout expired, but first Ignite daemon is still not up and running" + else + # If node isn't the first node of Ignite cluster and it failed to start we need to + # remove cluster-join lock to allow other EC2 nodes to acquire it + removeClusterJoinLock + + # If node failed all SERVICE_START_ATTEMPTS attempts to start Ignite daemon we will not + # try anymore and terminate with error + if [ $START_ATTEMPT -gt $SERVICE_START_ATTEMPTS ]; then + terminate "${SERVICE_START_ATTEMPTS} attempts exceed, but Ignite daemon is still not up and running" + fi + + # New attempt to start Ignite daemon + startIgnite + fi + + continue + fi + + # Handling situation when Ignite daemon process abnormally terminated + if [ -z "$proc" ]; then + # If this is the first node of Ignite cluster just terminating with error + if [ "$FIRST_NODE_LOCK" == "true" ]; then + terminate "Failed to start Ignite daemon" + fi + + # Remove cluster-join lock to allow other EC2 nodes to acquire it + removeClusterJoinLock + + echo "[WARN] Failed to start Ignite daemon. Sleeping for extra 30sec" + sleep 30s + + # New attempt to start Ignite daemon + startIgnite + + continue + fi + + echo "[INFO] Waiting for Ignite daemon to start, time passed ${duration}min" + sleep 30s +done + +# Once Ignite daemon successfully started we registering new Ignite node in S3 +registerNode + +# Terminating script with zero exit code +terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/logs-collector.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/logs-collector.sh b/modules/cassandra/store/src/test/bootstrap/aws/logs-collector.sh new file mode 100644 index 0000000..1634b89 --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/logs-collector.sh @@ -0,0 +1,173 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Logs collector daemon +# ----------------------------------------------------------------------------------------------- +# Script is launched in background by all EC2 nodes of all clusters (Cassandra, Ignite, Tests) and +# periodically (each 30 seconds) checks if specific S3 trigger file (specified by $S3_LOGS_TRIGGER_URL) +# was created or its timestamp was changed. Such an event serve as a trigger for the script +# to collect EC2 instance logs (from folder specified by $1) and upload them into specific +# S3 folder (specified by $S3_LOGS_FOLDER). +# ----------------------------------------------------------------------------------------------- + +uploadLogs() +{ + if [ ! -d "$1" ]; then + echo "[INFO] Logs directory doesn't exist: $1" + return 0 + fi + + echo "[INFO] Uploading logs from directory: $1" + + dirList=$(ls $1 | head -1) + + if [ -z "$dirList" ]; then + echo "[INFO] Directory is empty: $1" + fi + + for i in 0 9; + do + aws s3 sync --sse AES256 --delete "$1" "$S3_LOGS_FOLDER" + code=$? + + if [ $code -eq 0 ]; then + echo "[INFO] Successfully uploaded logs from directory: $1" + return 0 + fi + + echo "[WARN] Failed to upload logs from $i attempt, sleeping extra 30sec" + sleep 30s + done + + echo "[ERROR] All 10 attempts to upload logs are failed for the directory: $1" +} + +createNewLogsSnapshot() +{ + rm -f ~/logs-collector.snapshot.new + + for log_src in "$@" + do + if [ -d "$log_src" ] || [ -f "$log_src" ]; then + ls -alR $log_src >> ~/logs-collector.snapshot.new + + fi + done +} + +checkLogsChanged() +{ + createNewLogsSnapshot $@ + + if [ ! -f "~/logs-collector.snapshot" ]; then + return 1 + fi + + diff "~/logs-collector.snapshot" "~/logs-collector.snapshot.new" > /dev/null + + return $? +} + +updateLogsSnapshot() +{ + if [ ! -f "~/logs-collector.snapshot.new" ]; then + return 0 + fi + + rm -f "~/logs-collector.snapshot" + mv "~/logs-collector.snapshot.new" "~/logs-collector.snapshot" +} + +collectLogs() +{ + createNewLogsSnapshot + + rm -Rf ~/logs-collector-logs + mkdir -p ~/logs-collector-logs + + for log_src in "$@" + do + if [ -f "$log_src" ]; then + echo "[INFO] Collecting log file: $log_src" + cp -f $log_src ~/logs-collector-logs + elif [ -d "$log_src" ]; then + echo "[INFO] Collecting logs from folder: $log_src" + cp -Rf $log_src ~/logs-collector-logs + fi + done + + uploadLogs ~/logs-collector-logs + + rm -Rf ~/logs-collector-logs + + updateLogsSnapshot +} + +echo "[INFO] Running Logs collector service" + +if [ -z "$1" ]; then + echo "[ERROR] Logs collection S3 trigger URL doesn't specified" + exit 1 +fi + +S3_LOGS_TRIGGER_URL=$1 + +echo "[INFO] Logs collection S3 trigger URL: $S3_LOGS_TRIGGER_URL" + +if [ -z "$2" ]; then + echo "[ERROR] S3 folder where to upload logs doesn't specified" + exit 1 +fi + +S3_LOGS_FOLDER=$2 + +echo "[INFO] S3 logs upload folder: $S3_LOGS_FOLDER" + +shift 2 + +if [ -z "$1" ]; then + echo "[WARN] Local logs sources don't specified" +else + echo "[INFO] Local logs sources: $@" +fi + +echo "--------------------------------------------------------------------" + +TRIGGER_STATE= + +while true; do + sleep 30s + + STATE=$(aws s3 ls $S3_LOGS_TRIGGER_URL) + + if [ -z "$STATE" ] || [ "$STATE" == "$TRIGGER_STATE" ]; then + checkLogsChanged + + if [ $? -eq 0 ]; then + continue + fi + fi + + TRIGGER_STATE=$STATE + + collectLogs $@ /var/log/cloud-init.log /var/log/cloud-init-output.log + + echo "--------------------------------------------------------------------" +done http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml b/modules/cassandra/store/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml new file mode 100644 index 0000000..77ffb9e --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/tests/ignite-cassandra-client-template.xml @@ -0,0 +1,192 @@ +<?xml version="1.0" encoding="UTF-8"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<beans xmlns="http://www.springframework.org/schema/beans"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util"; + xsi:schemaLocation="http://www.springframework.org/schema/beans + http://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/util + http://www.springframework.org/schema/util/spring-util.xsd";> + + <bean id="loadBalancingPolicy" class="com.datastax.driver.core.policies.TokenAwarePolicy"> + <constructor-arg type="com.datastax.driver.core.policies.LoadBalancingPolicy"> + <bean class="com.datastax.driver.core.policies.RoundRobinPolicy"/> + </constructor-arg> + </bean> + + <util:list id="contactPoints" value-type="java.lang.String"> + ${CASSANDRA_SEEDS} + </util:list> + + <bean id="cassandraAdminDataSource" class="org.apache.ignite.cache.store.cassandra.datasource.DataSource"> + <property name="user" value="cassandra"/> + <property name="password" value="cassandra"/> + <property name="contactPoints" ref="contactPoints"/> + <property name="readConsistency" value="ONE"/> + <property name="writeConsistency" value="ONE"/> + <property name="loadBalancingPolicy" ref="loadBalancingPolicy"/> + </bean> + + <!-- Persistence settings for 'cache1' --> + <bean id="cache1_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> + <constructor-arg type="java.lang.String"> + <value><![CDATA[ +<persistence keyspace="test1" table="primitive_test1"> + <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" column="key"/> + <valuePersistence class="java.lang.Long" strategy="PRIMITIVE" column="value"/> +</persistence>]]> + </value> + </constructor-arg> + </bean> + + <!-- Persistence settings for 'cache2' --> + <bean id="cache2_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> + <constructor-arg type="java.lang.String"> + <value><![CDATA[ +<persistence keyspace="test1" table="blob_test1"> + <keyPersistence class="java.lang.Long" strategy="PRIMITIVE" /> + <valuePersistence strategy="BLOB"/> +</persistence>]]> + </value> + </constructor-arg> + </bean> + + <!-- Persistence settings for 'cache3' --> + <bean id="cache3_settings" class="org.apache.ignite.cache.store.cassandra.persistence.KeyValuePersistenceSettings"> + <constructor-arg type="java.lang.String"> + <value><![CDATA[ +<persistence keyspace="test1" table="pojo_test3" ttl="86400"> + <keyspaceOptions> + REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 3} + AND DURABLE_WRITES = true + </keyspaceOptions> + <tableOptions> + comment = 'A most excellent and useful table' + AND read_repair_chance = 0.2 + </tableOptions> + <keyPersistence class="org.apache.ignite.tests.pojos.PersonId" strategy="POJO"> + <partitionKey> + <field name="companyCode" column="company" /> + <field name="departmentCode" column="department" /> + </partitionKey> + <clusterKey> + <field name="personNumber" column="number" sort="desc"/> + </clusterKey> + </keyPersistence> + <valuePersistence class="org.apache.ignite.tests.pojos.Person" + strategy="POJO" + serializer="org.apache.ignite.cache.store.cassandra.serializer.KryoSerializer"> + <field name="firstName" column="first_name" /> + <field name="lastName" column="last_name" /> + <field name="age" /> + <field name="married" index="true"/> + <field name="height" /> + <field name="weight" /> + <field name="birthDate" column="birth_date" /> + <field name="phones" /> + </valuePersistence> +</persistence>]]> + </value> + </constructor-arg> + </bean> + + <!-- Ignite configuration --> + <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> + <!-- Turn on client mode --> + <property name="clientMode" value="true"/> + + <property name="metricsLogFrequency" value="0"/> + + <property name="cacheConfiguration"> + <list> + <!-- Configuring persistence for "cache1" cache --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="cache1"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> + <property name="readThrough" value="true"/> + <property name="writeThrough" value="true"/> + <property name="writeBehindEnabled" value="true"/> + <property name="cacheStoreFactory"> + <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> + <property name="dataSourceBean" value="cassandraAdminDataSource"/> + <property name="persistenceSettingsBean" value="cache1_settings"/> + </bean> + </property> + </bean> + + <!-- Configuring persistence for "cache2" cache --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="cache2"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> + <property name="readThrough" value="true"/> + <property name="writeThrough" value="true"/> + <property name="writeBehindEnabled" value="true"/> + <property name="cacheStoreFactory"> + <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> + <property name="dataSourceBean" value="cassandraAdminDataSource"/> + <property name="persistenceSettingsBean" value="cache2_settings"/> + </bean> + </property> + </bean> + + <!-- Configuring persistence for "cache3" cache --> + <bean class="org.apache.ignite.configuration.CacheConfiguration"> + <property name="name" value="cache3"/> + <property name="startSize" value="1000000"/> + <property name="cacheMode" value="PARTITIONED"/> + <property name="backups" value="0"/> + <property name="offHeapMaxMemory" value="0"/> + <property name="swapEnabled" value="false"/> + <property name="readThrough" value="true"/> + <property name="writeThrough" value="true"/> + <property name="writeBehindEnabled" value="true"/> + <property name="cacheStoreFactory"> + <bean class="org.apache.ignite.cache.store.cassandra.CassandraCacheStoreFactory"> + <property name="dataSourceBean" value="cassandraAdminDataSource"/> + <property name="persistenceSettingsBean" value="cache3_settings"/> + </bean> + </property> + </bean> + </list> + </property> + + <!-- Configuring remote ignite cluster connections --> + <property name="discoverySpi"> + <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> + <property name="ipFinder"> + <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.multicast.TcpDiscoveryMulticastIpFinder"> + <property name="addresses"> + <list> + ${IGNITE_SEEDS} + </list> + </property> + </bean> + </property> + </bean> + </property> + </bean> +</beans> http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-bootstrap.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-bootstrap.sh b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-bootstrap.sh new file mode 100644 index 0000000..8e6faff --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-bootstrap.sh @@ -0,0 +1,317 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Bootstrap script to spin up Tests cluster +# ----------------------------------------------------------------------------------------------- + +# URL to download AWS CLI tools +AWS_CLI_DOWNLOAD_URL=https://s3.amazonaws.com/aws-cli/awscli-bundle.zip + +# URL to download JDK +JDK_DOWNLOAD_URL=http://download.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.tar.gz + +# URL to download Ignite-Cassandra tests package - you should previously package and upload it to this place +TESTS_PACKAGE_DONLOAD_URL=s3://<bucket>/<folder>/ignite-cassandra-tests-<version>.zip + +# Terminates script execution and upload logs to S3 +terminate() +{ + SUCCESS_URL=$S3_TESTS_SUCCESS + FAILURE_URL=$S3_TESTS_FAILURE + + if [ -n "$SUCCESS_URL" ] && [[ "$SUCCESS_URL" != */ ]]; then + SUCCESS_URL=${SUCCESS_URL}/ + fi + + if [ -n "$FAILURE_URL" ] && [[ "$FAILURE_URL" != */ ]]; then + FAILURE_URL=${FAILURE_URL}/ + fi + + host_name=$(hostname -f | tr '[:upper:]' '[:lower:]') + msg=$host_name + + if [ -n "$1" ]; then + echo "[ERROR] $1" + echo "[ERROR]-----------------------------------------------------" + echo "[ERROR] Test node bootstrap failed" + echo "[ERROR]-----------------------------------------------------" + msg=$1 + + if [ -z "$FAILURE_URL" ]; then + exit 1 + fi + + reportFolder=${FAILURE_URL}${host_name} + reportFile=$reportFolder/__error__ + else + echo "[INFO]-----------------------------------------------------" + echo "[INFO] Test node bootstrap successfully completed" + echo "[INFO]-----------------------------------------------------" + + if [ -z "$SUCCESS_URL" ]; then + exit 0 + fi + + reportFolder=${SUCCESS_URL}${host_name} + reportFile=$reportFolder/__success__ + fi + + echo $msg > /opt/bootstrap-result + + aws s3 rm --recursive $reportFolder + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to drop report folder: $reportFolder" + fi + + aws s3 cp --sse AES256 /opt/bootstrap-result $reportFile + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to report bootstrap result to: $reportFile" + fi + + rm -f /opt/bootstrap-result + + if [ -n "$1" ]; then + exit 1 + fi + + exit 0 +} + +# Downloads specified package +downloadPackage() +{ + echo "[INFO] Downloading $3 package from $1 into $2" + + for i in 0 9; + do + if [[ "$1" == s3* ]]; then + aws s3 cp $1 $2 + code=$? + else + curl "$1" -o "$2" + code=$? + fi + + if [ $code -eq 0 ]; then + echo "[INFO] $3 package successfully downloaded from $1 into $2" + return 0 + fi + + echo "[WARN] Failed to download $3 package from $i attempt, sleeping extra 5sec" + sleep 5s + done + + terminate "All 10 attempts to download $3 package from $1 are failed" +} + +# Downloads and setup JDK +setupJava() +{ + rm -Rf /opt/java /opt/jdk.tar.gz + + echo "[INFO] Downloading 'jdk'" + wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com%2F; oraclelicense=accept-securebackup-cookie" "$JDK_DOWNLOAD_URL" -O /opt/jdk.tar.gz + if [ $? -ne 0 ]; then + terminate "Failed to download 'jdk'" + fi + + echo "[INFO] Untaring 'jdk'" + tar -xvzf /opt/jdk.tar.gz -C /opt + if [ $? -ne 0 ]; then + terminate "Failed to untar 'jdk'" + fi + + rm -Rf /opt/jdk.tar.gz + + unzipDir=$(ls /opt | grep "jdk") + if [ "$unzipDir" != "java" ]; then + mv /opt/$unzipDir /opt/java + fi +} + +# Downloads and setup AWS CLI +setupAWSCLI() +{ + echo "[INFO] Installing 'awscli'" + pip install --upgrade awscli + if [ $? -eq 0 ]; then + return 0 + fi + + echo "[ERROR] Failed to install 'awscli' using pip" + echo "[INFO] Trying to install awscli using zip archive" + echo "[INFO] Downloading awscli zip" + + downloadPackage "$AWS_CLI_DOWNLOAD_URL" "/opt/awscli-bundle.zip" "awscli" + + echo "[INFO] Unzipping awscli zip" + unzip /opt/awscli-bundle.zip -d /opt + if [ $? -ne 0 ]; then + terminate "Failed to unzip awscli zip" + fi + + rm -Rf /opt/awscli-bundle.zip + + echo "[INFO] Installing awscli" + /opt/awscli-bundle/install -i /usr/local/aws -b /usr/local/bin/aws + if [ $? -ne 0 ]; then + terminate "Failed to install awscli" + fi + + echo "[INFO] Successfully installed awscli from zip archive" +} + +# Setup all the pre-requisites (packages, settings and etc.) +setupPreRequisites() +{ + echo "[INFO] Installing 'wget' package" + yum -y install wget + if [ $? -ne 0 ]; then + terminate "Failed to install 'wget' package" + fi + + echo "[INFO] Installing 'net-tools' package" + yum -y install net-tools + if [ $? -ne 0 ]; then + terminate "Failed to install 'net-tools' package" + fi + + echo "[INFO] Installing 'python' package" + yum -y install python + if [ $? -ne 0 ]; then + terminate "Failed to install 'python' package" + fi + + echo "[INFO] Installing 'unzip' package" + yum -y install unzip + if [ $? -ne 0 ]; then + terminate "Failed to install 'unzip' package" + fi + + downloadPackage "https://bootstrap.pypa.io/get-pip.py"; "/opt/get-pip.py" "get-pip.py" + + echo "[INFO] Installing 'pip'" + python /opt/get-pip.py + if [ $? -ne 0 ]; then + terminate "Failed to install 'pip'" + fi +} + +# Downloads and setup tests package +setupTestsPackage() +{ + downloadPackage "$TESTS_PACKAGE_DONLOAD_URL" "/opt/ignite-cassandra-tests.zip" "Tests" + + rm -Rf /opt/ignite-cassandra-tests + + unzip /opt/ignite-cassandra-tests.zip -d /opt + if [ $? -ne 0 ]; then + terminate "Failed to unzip tests package" + fi + + rm -f /opt/ignite-cassandra-tests.zip + + unzipDir=$(ls /opt | grep "ignite-cassandra") + if [ "$unzipDir" != "ignite-cassandra-tests" ]; then + mv /opt/$unzipDir /opt/ignite-cassandra-tests + fi + + find /opt/ignite-cassandra-tests -type f -name "*.sh" -exec chmod ug+x {} \; + + . /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" + + setupNTP + + echo "[INFO] Starting logs collector daemon" + + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + /opt/ignite-cassandra-tests/bootstrap/aws/logs-collector.sh "$S3_LOGS_TRIGGER" "$S3_TESTS_LOGS/$HOST_NAME" "/opt/ignite-cassandra-tests/logs" > /opt/logs-collector.log & + + echo "[INFO] Logs collector daemon started: $!" + + echo "----------------------------------------------------------------------------------------" + printInstanceInfo + echo "----------------------------------------------------------------------------------------" + tagInstance + bootstrapGangliaAgent "test" 8643 + + ################################################### + # Extra configuration specific only for test node # + ################################################### + + echo "[INFO] Installing bc package" + + yum -y install bc + + if [ $? -ne 0 ]; then + terminate "Failed to install bc package" + fi + + echo "[INFO] Installing zip package" + + yum -y install zip + + if [ $? -ne 0 ]; then + terminate "Failed to install zip package" + fi + + echo "[INFO] Creating 'ignite' group" + exists=$(cat /etc/group | grep ignite) + if [ -z "$exists" ]; then + groupadd ignite + if [ $? -ne 0 ]; then + terminate "Failed to create 'ignite' group" + fi + fi + + echo "[INFO] Creating 'ignite' user" + exists=$(cat /etc/passwd | grep ignite) + if [ -z "$exists" ]; then + useradd -g ignite ignite + if [ $? -ne 0 ]; then + terminate "Failed to create 'ignite' user" + fi + fi + + mkdir -p /opt/ignite-cassandra-tests/logs + chown -R ignite:ignite /opt/ignite-cassandra-tests + + echo "export JAVA_HOME=/opt/java" >> $1 + echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> $1 +} + +################################################################################################################### + +echo "[INFO]-----------------------------------------------------------------" +echo "[INFO] Bootstrapping Tests node" +echo "[INFO]-----------------------------------------------------------------" + +setupPreRequisites +setupJava +setupAWSCLI +setupTestsPackage "/root/.bash_profile" + +cmd="/opt/ignite-cassandra-tests/bootstrap/aws/tests/tests-manager.sh" + +#sudo -u ignite -g ignite sh -c "$cmd > /opt/ignite-cassandra-tests/tests-manager" & + +$cmd > /opt/ignite-cassandra-tests/logs/tests-manager.log & + +terminate \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-manager.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-manager.sh b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-manager.sh new file mode 100644 index 0000000..c0f5d6b --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-manager.sh @@ -0,0 +1,458 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Tests manager daemon +# ----------------------------------------------------------------------------------------------- +# Script is launched in background by all nodes of Tests cluster and +# periodically (each 30 seconds) checks if specific S3 trigger file was created or +# its timestamp was changed. Such an event serve as a trigger for the script to start +# preparing to run load tests. +# ----------------------------------------------------------------------------------------------- + +#profile=/home/ignite/.bash_profile +profile=/root/.bash_profile + +. $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" + +# Switch test node to IDLE state +switchToIdleState() +{ + if [ "$NODE_STATE" != "IDLE" ]; then + echo "[INFO] Switching node to IDLE state" + dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_IDLE" + NODE_STATE="IDLE" + echo "[INFO] Node was switched to IDLE state" + fi +} + +# Switch test node to PREPARING state +switchToPreparingState() +{ + if [ "$NODE_STATE" != "PREPARING" ]; then + echo "[INFO] Switching node to PREPARING state" + dropStateFlag "$S3_TESTS_WAITING" "$S3_TESTS_IDLE" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_PREPARING" + NODE_STATE="PREPARING" + echo "[INFO] Node was switched to PREPARING state" + fi +} + +# Switch test node to WAITING state +switchToWaitingState() +{ + if [ "$NODE_STATE" != "WAITING" ]; then + echo "[INFO] Switching node to WAITING state" + dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_RUNNING" + createStateFlag "$S3_TESTS_WAITING" + NODE_STATE="WAITING" + echo "[INFO] Node was switched to WAITING state" + fi +} + +# Switch test node to RUNNING state +switchToRunningState() +{ + if [ "$NODE_STATE" != "RUNNING" ]; then + echo "[INFO] Switching node to RUNNING state" + dropStateFlag "$S3_TESTS_IDLE" "$S3_TESTS_PREPARING" "$S3_TESTS_WAITING" + createStateFlag "$S3_TESTS_RUNNING" + NODE_STATE="RUNNING" + echo "[INFO] Node was switched to RUNNING state" + fi +} + +# Creates appropriate state flag for the node in S3 +createStateFlag() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + aws s3 cp --sse AES256 /etc/hosts ${1}${HOST_NAME} + if [ $? -ne 0 ]; then + terminate "Failed to create state flag: ${1}${HOST_NAME}" + fi +} + +# Drops appropriate state flag for the node in S3 +dropStateFlag() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + for flagUrl in "$@" + do + exists=$(aws s3 ls ${flagUrl}${HOST_NAME}) + if [ -n "$exists" ]; then + aws s3 rm ${flagUrl}${HOST_NAME} + if [ $? -ne 0 ]; then + terminate "Failed to drop state flag: ${flagUrl}${HOST_NAME}" + fi + fi + done +} + +# Removes tests summary report from S3 +dropTestsSummary() +{ + exists=$(aws s3 ls $S3_TESTS_SUMMARY) + if [ -z "$exists" ]; then + return 0 + fi + + aws s3 rm $S3_TESTS_SUMMARY + if [ $? -ne 0 ]; then + terminate "Failed to drop tests summary info: $S3_TESTS_SUMMARY" + fi +} + +# Recreate all the necessary Cassandra artifacts before running Load tests +recreateCassandraArtifacts() +{ + /opt/ignite-cassandra-tests/recreate-cassandra-artifacts.sh + if [ $? -ne 0 ]; then + terminate "Failed to recreate Cassandra artifacts" + fi +} + +# Setups Cassandra seeds for this Tests node being able to connect to Cassandra. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. +setupCassandraSeeds() +{ + if [ $CASSANDRA_NODES_COUNT -eq 0 ]; then + return 0 + fi + + setupClusterSeeds "cassandra" + + CASSANDRA_SEEDS1=$(echo $CLUSTER_SEEDS | sed -r "s/ /,/g") + CASSANDRA_SEEDS2= + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + CASSANDRA_SEEDS2="${CASSANDRA_SEEDS2}<value>$seed<\/value>" + done + + echo "[INFO] Using Cassandra seeds: $CASSANDRA_SEEDS1" + + echo "contact.points=$CASSANDRA_SEEDS1" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/connection.properties + + cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template.xml | sed -r "s/\\\$\{CASSANDRA_SEEDS\}/$CASSANDRA_SEEDS2/g" > /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml +} + +# Setups Ignite nodes for this Tests node being able to connect to Ignite. +# Looks for the information in S3 about already up and running Cassandra cluster nodes. +setupIgniteSeeds() +{ + if [ $IGNITE_NODES_COUNT -eq 0 ]; then + return 0 + fi + + setupClusterSeeds "ignite" + + CLUSTER_SEEDS=($CLUSTER_SEEDS) + count=${#CLUSTER_SEEDS[@]} + + IGNITE_SEEDS= + + for (( i=0; i<=$(( $count -1 )); i++ )) + do + seed=${CLUSTER_SEEDS[$i]} + IGNITE_SEEDS="${IGNITE_SEEDS}<value>$seed<\/value>" + done + + echo "[INFO] Using Ignite seeds: $IGNITE_SEEDS" + + cat /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml | sed -r "s/\\\$\{IGNITE_SEEDS\}/$IGNITE_SEEDS/g" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/persistence/primitive/ignite-remote-client-config.xml + rm -f /opt/ignite-cassandra-tests/bootstrap/aws/tests/ignite-cassandra-client-template1.xml +} + +# Setups Cassandra credentials to connect to Cassandra cluster +setupCassandraCredentials() +{ + echo "admin.user=cassandra" > /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "admin.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "regular.user=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties + echo "regular.password=cassandra" >> /opt/ignite-cassandra-tests/settings/org/apache/ignite/tests/cassandra/credentials.properties +} + +# Triggering first time tests execution for all nodes in the Tests cluster +triggerFirstTimeTestsExecution() +{ + if [ -z "$TESTS_TYPE" ]; then + return 0 + fi + + tryToGetFirstNodeLock + if [ $? -ne 0 ]; then + return 0 + fi + + sleep 30s + + echo "[INFO] Triggering first time tests execution" + + echo "TESTS_TYPE=$TESTS_TYPE" > /opt/ignite-cassandra-tests/tests-trigger + echo "#--------------------------------------------------" >> /opt/ignite-cassandra-tests/tests-trigger + echo "" >> /opt/ignite-cassandra-tests/tests-trigger + cat /opt/ignite-cassandra-tests/settings/tests.properties >> /opt/ignite-cassandra-tests/tests-trigger + + aws s3 cp --sse AES256 /opt/ignite-cassandra-tests/tests-trigger $S3_TESTS_TRIGGER + code=$? + + rm -f /opt/ignite-cassandra-tests/tests-trigger + + if [ $code -ne 0 ]; then + terminate "Failed to create tests trigger: $S3_TESTS_TRIGGER" + fi +} + +# Cleans previously created logs from S3 +cleanPreviousLogs() +{ + for logFile in /opt/ignite-cassandra-tests/logs/* + do + managerLog=$(echo $logFile | grep "tests-manager") + if [ -z "$managerLog" ]; then + rm -Rf $logFile + fi + done + + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + aws s3 rm --recursive ${S3_TESTS_FAILURE}${HOST_NAME} + aws s3 rm --recursive ${S3_TESTS_SUCCESS}${HOST_NAME} +} + +# Uploads tests logs to S3 +uploadTestsLogs() +{ + HOST_NAME=$(hostname -f | tr '[:upper:]' '[:lower:]') + + if [ -f "/opt/ignite-cassandra-tests/logs/__success__" ]; then + logsFolder=${S3_TESTS_SUCCESS}${HOST_NAME} + else + logsFolder=${S3_TESTS_FAILURE}${HOST_NAME} + fi + + aws s3 rm --recursive $logsFolder + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to drop logs folder: $logsFolder" + fi + + if [ -d "/opt/ignite-cassandra-tests/logs" ]; then + aws s3 sync --sse AES256 /opt/ignite-cassandra-tests/logs $logsFolder + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to export tests logs to: $logsFolder" + fi + fi +} + +# Runs tests-report.sh to prepare tests summary report +buildTestsSummaryReport() +{ + reportScript=$(readlink -m $( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/tests-report.sh) + $reportScript + + if [ -n "$S3_LOGS_TRIGGER" ]; then + aws s3 cp --sse AES256 /etc/hosts $S3_LOGS_TRIGGER + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to trigger logs collection" + fi + fi +} + +# Running load tests +runLoadTests() +{ + cd /opt/ignite-cassandra-tests + + if [ "$TESTS_TYPE" == "ignite" ]; then + echo "[INFO] Running Ignite load tests" + ./ignite-load-tests.sh & + else + echo "[INFO] Running Cassandra load tests" + ./cassandra-load-tests.sh & + fi + + testsJobId=$! + + echo "[INFO] Tests job id: $testsJobId" + + sleep 1m + + LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) + LOGS_SNAPSHOT_TIME=$(date +%s) + + TERMINATED= + + # tests monitoring + while true; do + proc=$(ps -ef | grep java | grep "org.apache.ignite.tests") + if [ -z "$proc" ]; then + break + fi + + NEW_LOGS_SNAPSHOT=$(ls -al /opt/ignite-cassandra-tests/logs) + NEW_LOGS_SNAPSHOT_TIME=$(date +%s) + + # if logs state updated it means that tests are running and not stuck + if [ "$LOGS_SNAPSHOT" != "$NEW_LOGS_SNAPSHOT" ]; then + LOGS_SNAPSHOT=$NEW_LOGS_SNAPSHOT + LOGS_SNAPSHOT_TIME=$NEW_LOGS_SNAPSHOT_TIME + continue + fi + + duration=$(( $NEW_LOGS_SNAPSHOT_TIME-$LOGS_SNAPSHOT_TIME )) + duration=$(( $duration/60 )) + + # if logs wasn't updated during 5min it means that load tests stuck + if [ $duration -gt 5 ]; then + proc=($proc) + kill -9 ${proc[1]} + TERMINATED="true" + break + fi + + echo "[INFO] Waiting extra 30sec for load tests to complete" + + sleep 30s + done + + rm -f /opt/ignite-cassandra-tests/logs/tests.properties + cp /opt/ignite-cassandra-tests/settings/tests.properties /opt/ignite-cassandra-tests/logs + + if [ "$TERMINATED" == "true" ]; then + echo "[ERROR] Load tests stuck, tests process terminated" + echo "Load tests stuck, tests process terminated" > /opt/ignite-cassandra-tests/logs/__error__ + return 0 + fi + + failed= + if [ "$TESTS_TYPE" == "cassandra" ]; then + failed=$(cat /opt/ignite-cassandra-tests/cassandra-load-tests.log | grep "load tests execution failed") + else + failed=$(cat /opt/ignite-cassandra-tests/ignite-load-tests.log | grep "load tests execution failed") + fi + + if [ -n "$failed" ]; then + echo "[ERROR] Load tests execution failed" + echo "Load tests execution failed" > /opt/ignite-cassandra-tests/logs/__error__ + else + echo "[INFO] Load tests execution successfully completed" + echo "Load tests execution successfully completed" > /opt/ignite-cassandra-tests/logs/__success__ + fi +} + +####################################################################################################### + +sleep 1m + +NODE_STATE= +TRIGGER_STATE= + +printInstanceInfo +setupCassandraCredentials +switchToIdleState + +triggerFirstTimeTestsExecution + +registerNode + +while true; do + # switching state to IDLE + switchToIdleState + + sleep 30s + + NEW_TRIGGER_STATE=$(aws s3 ls $S3_TESTS_TRIGGER | xargs) + if [ -z "$NEW_TRIGGER_STATE" ] || [ "$NEW_TRIGGER_STATE" == "$TRIGGER_STATE" ]; then + continue + fi + + echo "----------------------------------------------------------------------" + echo "[INFO] Tests trigger changed" + echo "----------------------------------------------------------------------" + echo "[INFO] Old trigger: $TRIGGER_STATE" + echo "----------------------------------------------------------------------" + echo "[INFO] New trigger: $NEW_TRIGGER_STATE" + echo "----------------------------------------------------------------------" + + TRIGGER_STATE=$NEW_TRIGGER_STATE + + aws s3 cp $S3_TESTS_TRIGGER /opt/ignite-cassandra-tests/tests-trigger + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to download tests trigger info from: $S3_TESTS_TRIGGER" + continue + fi + + TESTS_TYPE=$(cat /opt/ignite-cassandra-tests/tests-trigger | grep TESTS_TYPE | xargs | sed -r "s/TESTS_TYPE=//g") + if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then + rm -f /opt/ignite-cassandra-tests/tests-trigger + echo "[ERROR] Incorrect tests type specified in the trigger info: $S3_TESTS_TRIGGER" + continue + fi + + rm -f /opt/ignite-cassandra-tests/settings/tests.properties + mv -f /opt/ignite-cassandra-tests/tests-trigger /opt/ignite-cassandra-tests/settings/tests.properties + + waitAllTestNodesCompletedTests + + # switching state to PREPARING + switchToPreparingState + + waitAllClusterNodesReady "cassandra" + waitAllClusterNodesReady "ignite" + setupCassandraSeeds + setupIgniteSeeds + + cleanPreviousLogs + + tryToGetFirstNodeLock + if [ $? -eq 0 ]; then + dropTestsSummary + recreateCassandraArtifacts + fi + + # switching state to WAITING + switchToWaitingState + + waitAllClusterNodesReady "test" + + if [ "$FIRST_NODE_LOCK" == "true" ]; then + aws s3 rm $S3_TESTS_TRIGGER + fi + + # switching state to RUNNING + switchToRunningState + + runLoadTests + uploadTestsLogs + + tryToGetFirstNodeLock + if [ $? -eq 0 ]; then + waitAllTestNodesCompletedTests + buildTestsSummaryReport + removeFirstNodeLock + fi +done \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-report.sh ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-report.sh b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-report.sh new file mode 100644 index 0000000..1576d57 --- /dev/null +++ b/modules/cassandra/store/src/test/bootstrap/aws/tests/tests-report.sh @@ -0,0 +1,499 @@ +#!/bin/sh + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------------------------------- +# Tests report builder +# ----------------------------------------------------------------------------------------------- +# Script is used to analyze load tests logs collected from all 'Tests' cluster nodes and build +# summary report +# ----------------------------------------------------------------------------------------------- + +#profile=/home/ignite/.bash_profile +profile=/root/.bash_profile + +. $profile +. /opt/ignite-cassandra-tests/bootstrap/aws/common.sh "test" + +# Building tests summary report +reportTestsSummary() +{ + echo "[INFO] Preparing tests results summary" + + TESTS_SUMMARY_DIR=/opt/ignite-cassandra-tests/tests-summary + SUCCEED_NODES_FILE=$TESTS_SUMMARY_DIR/succeed-nodes + SUCCEED_NODES_DIR=$TESTS_SUMMARY_DIR/succeed + FAILED_NODES_FILE=$TESTS_SUMMARY_DIR/failed-nodes + FAILED_NODES_DIR=$TESTS_SUMMARY_DIR/failed + REPORT_FILE=$TESTS_SUMMARY_DIR/report.txt + + rm -Rf $TESTS_SUMMARY_DIR + mkdir -p $TESTS_SUMMARY_DIR + mkdir -p $SUCCEED_NODES_DIR + mkdir -p $FAILED_NODES_DIR + + aws s3 ls $S3_TESTS_SUCCESS | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $SUCCEED_NODES_FILE + aws s3 ls $S3_TESTS_FAILURE | sed -r "s/PRE //g" | sed -r "s/ //g" | sed -r "s/\///g" > $FAILED_NODES_FILE + + succeedCount=$(cat $SUCCEED_NODES_FILE | wc -l) + failedCount=$(cat $FAILED_NODES_FILE | wc -l) + count=$(( $succeedCount+$failedCount )) + + echo "Test type : $TESTS_TYPE" > $REPORT_FILE + echo "Test nodes count : $count" >> $REPORT_FILE + echo "Test nodes succeed: $succeedCount" >> $REPORT_FILE + echo "Test nodes failed : $failedCount" >> $REPORT_FILE + echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE + + if [ $succeedCount -gt 0 ]; then + echo "Succeed test nodes |" >> $REPORT_FILE + echo "-------------------" >> $REPORT_FILE + cat $SUCCEED_NODES_FILE >> $REPORT_FILE + echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE + + aws s3 sync --delete $S3_TESTS_SUCCESS $SUCCEED_NODES_DIR + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to get succeed tests details" + else + reportSucceedTestsStatistics "$REPORT_FILE" "$SUCCEED_NODES_DIR" + fi + fi + + if [ $failedCount -gt 0 ]; then + echo "Failed test nodes |" >> $REPORT_FILE + echo "------------------" >> $REPORT_FILE + cat $FAILED_NODES_FILE >> $REPORT_FILE + echo "----------------------------------------------------------------------------------------------" >> $REPORT_FILE + + aws sync --delete $S3_TESTS_FAILURE $FAILED_NODES_DIR + if [ $? -ne 0 ]; then + echo "[ERROR] Failed to get failed tests details" + else + reportFailedTestsDetailes "$REPORT_FILE" "$FAILED_NODES_DIR" + fi + fi + + rm -f $HOME/tests-summary.zip + + pushd $TESTS_SUMMARY_DIR + + zip -r -9 $HOME/tests-summary.zip . + code=$? + + rm -Rf $TESTS_SUMMARY_DIR + + popd + + if [ $code -ne 0 ]; then + echo "-------------------------------------------------------------------------------------" + echo "[ERROR] Failed to create tests summary zip archive $HOME/tests-summary.zip for $TESTS_SUMMARY_DIR" + echo "-------------------------------------------------------------------------------------" + return 1 + fi + + aws s3 cp --sse AES256 $HOME/tests-summary.zip $S3_TESTS_SUMMARY + if [ $? -ne 0 ]; then + echo "-------------------------------------------------------------------------------------" + echo "[ERROR] Failed to uploat tests summary archive to: $S3_TESTS_SUMMARY" + echo "-------------------------------------------------------------------------------------" + else + echo "-------------------------------------------------------------------------------------" + echo "[INFO] Tests results summary uploaded to: $S3_TESTS_SUMMARY" + echo "-------------------------------------------------------------------------------------" + fi + + rm -f $HOME/tests-summary.zip +} + +# Creates report for succeed tests +reportSucceedTestsStatistics() +{ + writeMsg="0" + writeErrors="0" + writeSpeed="0" + blkWriteMsg="0" + blkWriteErrors="0" + blkWriteSpeed="0" + readMsg="0" + readErrors="0" + readSpeed="0" + blkReadMsg="0" + blkReadErrors="0" + blkReadSpeed="0" + + writeErrNodes= + blkWriteErrNodes= + readErrNodes= + blkReadErrNodes= + + tmpFile=`mktemp` + + for dir in $2/* + do + node=$(echo $dir | sed -r "s/^.*\///g") + echo "-------------------------------------------------------------------------------------" + echo "[INFO] Gathering statistics from $node test node" + echo "-------------------------------------------------------------------------------------" + + logFile=$(ls $dir | grep "${TESTS_TYPE}-load-tests.log" | head -1) + if [ -z "$logFile" ]; then + echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Node $node marked as succeeded," >> $tmpFile + echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + continue + fi + + logFile=$dir/$logFile + if [ ! -f "$logFile" ]; then + echo "[WARN] Node $node marked as succeeded, but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Node $node marked as succeeded," >> $tmpFile + echo "but it doesn't have \"${TESTS_TYPE}-load-tests.log\" tests results summary file" >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + continue + fi + + cnt=$(cat $logFile | grep "^WRITE messages" | sed -r "s/WRITE messages: //g" | xargs) + if [ -n "$cnt" ]; then + writeMsg=$(bc <<< "$writeMsg + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] WRITE messages: $cnt" + else + echo "[WARN] WRITE messages count is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect WRITE messages count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect WRITE messages count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^WRITE errors" | sed -r "s/WRITE errors: //g" | sed -r "s/,.*//g" | xargs) + if [ -n "$cnt" ]; then + echo "[INFO] WRITE errors: $cnt" + writeErrors=$(bc <<< "$writeErrors + $cnt") + if [ $cnt -ne 0 ]; then + if [ -n "$writeErrNodes" ]; then + writeErrNodes="${writeErrNodes}, " + fi + writeErrNodes="${writeErrNodes}${node}" + fi + else + echo "[WARN] Failed to detect WRITE errors count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect WRITE errors count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^WRITE speed" | sed -r "s/WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) + if [ -n "$cnt" ]; then + writeSpeed=$(bc <<< "$writeSpeed + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] WRITE speed: $cnt msg/sec" + else + echo "[WARN] WRITE speed is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "WRITE speed is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect WRITE speed for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect WRITE speed for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_WRITE messages" | sed -r "s/BULK_WRITE messages: //g" | xargs) + if [ -n "$cnt" ]; then + blkWriteMsg=$(bc <<< "$blkWriteMsg + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] BULK_WRITE messages: $cnt" + else + echo "[WARN] BULK_WRITE messages count is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "BULK_WRITE messages count is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect BULK_WRITE messages count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_WRITE messages count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_WRITE errors" | sed -r "s/BULK_WRITE errors: //g" | sed -r "s/,.*//g" | xargs) + if [ -n "$cnt" ]; then + blkWriteErrors=$(bc <<< "$blkWriteErrors + $cnt") + echo "[INFO] BULK_WRITE errors: $cnt" + if [ $cnt -ne 0 ]; then + if [ -n "$blkWriteErrNodes" ]; then + blkWriteErrNodes="${blkWriteErrNodes}, " + fi + blkWriteErrNodes="${blkWriteErrNodes}${node}" + fi + else + echo "[WARN] Failed to detect BULK_WRITE errors count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_WRITE errors count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_WRITE speed" | sed -r "s/BULK_WRITE speed: //g" | sed -r "s/ msg\/sec//g" | xargs) + if [ -n "$cnt" ]; then + blkWriteSpeed=$(bc <<< "$blkWriteSpeed + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] BULK_WRITE speed: $cnt msg/sec" + else + echo "[WARN] BULK_WRITE speed is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "BULK_WRITE speed is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect BULK_WRITE speed for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_WRITE speed for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^READ messages" | sed -r "s/READ messages: //g" | xargs) + if [ -n "$cnt" ]; then + readMsg=$(bc <<< "$readMsg + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] READ messages: $cnt" + else + echo "[WARN] READ messages count is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "READ messages count is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect READ messages count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect READ messages count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^READ errors" | sed -r "s/READ errors: //g" | sed -r "s/,.*//g" | xargs) + if [ -n "$cnt" ]; then + readErrors=$(bc <<< "$readErrors + $cnt") + echo "[INFO] READ errors: $cnt" + if [ $cnt -ne 0 ]; then + if [ -n "$readErrNodes" ]; then + blkWriteErrNodes="${readErrNodes}, " + fi + readErrNodes="${readErrNodes}${node}" + fi + else + echo "[WARN] Failed to detect READ errors count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect READ errors count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^READ speed" | sed -r "s/READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) + if [ -n "$cnt" ]; then + readSpeed=$(bc <<< "$readSpeed + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] READ speed: $cnt msg/sec" + else + echo "[WARN] READ speed is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "READ speed is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect READ speed for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect READ speed for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_READ messages" | sed -r "s/BULK_READ messages: //g" | xargs) + if [ -n "$cnt" ]; then + blkReadMsg=$(bc <<< "$blkReadMsg + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] BULK_READ messages: $cnt" + else + echo "[WARN] BULK_READ messages count is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "BULK_READ messages count is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect BULK_READ messages count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_READ messages count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_READ errors" | sed -r "s/BULK_READ errors: //g" | sed -r "s/,.*//g" | xargs) + if [ -n "$cnt" ]; then + blkReadErrors=$(bc <<< "$blkReadErrors + $cnt") + echo "[INFO] BULK_READ errors: $cnt" + if [ $cnt -ne 0 ]; then + if [ -n "$blkReadErrNodes" ]; then + blkReadErrNodes="${blkReadErrNodes}, " + fi + blkReadErrNodes="${blkReadErrNodes}${node}" + fi + else + echo "[WARN] Failed to detect BULK_READ errors count for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_READ errors count for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + + cnt=$(cat $logFile | grep "^BULK_READ speed" | sed -r "s/BULK_READ speed: //g" | sed -r "s/ msg\/sec//g" | xargs) + if [ -n "$cnt" ]; then + blkReadSpeed=$(bc <<< "$blkReadSpeed + $cnt") + if [ $cnt -ne 0 ]; then + echo "[INFO] BULK_READ speed: $cnt msg/sec" + else + echo "[WARN] BULK_READ speed is zero for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "BULK_READ speed is zero for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + else + echo "[WARN] Failed to detect BULK_READ speed for $node node. This test probably failed." + echo "WARNING |" >> $tmpFile + echo "--------" >> $tmpFile + echo "Failed to detect BULK_READ speed for $node node. This test probably failed." >> $tmpFile + echo "----------------------------------------------------------------------------------------------" >> $tmpFile + fi + done + + echo "-------------------------------------------------------------------------------------" + + echo "WRITE test metrics |" >> $1 + echo "-------------------" >> $1 + echo "Messages: $writeMsg" >> $1 + echo "Speed : $writeSpeed msg/sec" >> $1 + echo "Errors : $writeErrors" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + + echo "BULK_WRITE test metrics |" >> $1 + echo "------------------------" >> $1 + echo "Messages: $blkWriteMsg" >> $1 + echo "Speed : $blkWriteSpeed msg/sec" >> $1 + echo "Errors : $blkWriteErrors" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + + echo "READ test metrics |" >> $1 + echo "------------------" >> $1 + echo "Messages: $readMsg" >> $1 + echo "Speed : $readSpeed msg/sec" >> $1 + echo "Errors : $readErrors" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + + echo "BULK_READ test metrics |" >> $1 + echo "-----------------------" >> $1 + echo "Messages: $blkReadMsg" >> $1 + echo "Speed : $blkReadSpeed msg/sec" >> $1 + echo "Errors : $blkReadErrors" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + + if [ -n "$writeErrNodes" ]; then + echo "Nodes having WRITE errors |" >> $1 + echo "-------------------------------" >> $1 + echo "$writeErrNodes" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + fi + + if [ -n "$blkWriteErrNodes" ]; then + echo "Nodes having BULK_WRITE errors |" >> $1 + echo "-------------------------------" >> $1 + echo "$blkWriteErrNodes" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + fi + + if [ -n "$readErrNodes" ]; then + echo "Nodes having READ errors |" >> $1 + echo "-------------------------------" >> $1 + echo "$readErrNodes" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + fi + + if [ -n "$blkReadErrNodes" ]; then + echo "Nodes having BULK_READ errors |" >> $1 + echo "-------------------------------" >> $1 + echo "$blkReadErrNodes" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + fi + + cat $tmpFile >> $1 + + rm -f $tmpFile +} + +# Creates report for failed tests +reportFailedTestsDetailes() +{ + for dir in $2/* + do + node=$(echo $dir | sed -r "s/^.*\///g") + if [ -z "$node" ]; then + continue + fi + + echo "----------------------------------------------------------------------------------------------" >> $1 + echo "Error details for node: $node" >> $1 + echo "----------------------------------------------------------------------------------------------" >> $1 + + if [ -f "$dir/__error__" ]; then + cat $dir/__error__ >> $1 + else + echo "N/A" >> $1 + fi + done +} + +####################################################################################################### + +if [ "$TESTS_TYPE" != "ignite" ] && [ "$TESTS_TYPE" != "cassandra" ]; then + terminate "Incorrect tests type specified: $TESTS_TYPE" +fi + +reportTestsSummary \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ignite/blob/231ead01/modules/cassandra/store/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java ---------------------------------------------------------------------- diff --git a/modules/cassandra/store/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java b/modules/cassandra/store/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java new file mode 100644 index 0000000..c031bfb --- /dev/null +++ b/modules/cassandra/store/src/test/java/org/apache/ignite/tests/CassandraDirectPersistenceLoadTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.tests; + +import org.apache.ignite.tests.load.LoadTestDriver; +import org.apache.ignite.tests.load.cassandra.BulkReadWorker; +import org.apache.ignite.tests.load.cassandra.BulkWriteWorker; +import org.apache.ignite.tests.load.cassandra.ReadWorker; +import org.apache.ignite.tests.load.cassandra.WriteWorker; +import org.apache.ignite.tests.utils.CacheStoreHelper; +import org.apache.ignite.tests.utils.CassandraHelper; +import org.apache.ignite.tests.utils.TestsHelper; +import org.apache.log4j.Logger; + +/** + * Load tests for {@link org.apache.ignite.cache.store.cassandra.CassandraCacheStore} implementation of + * {@link org.apache.ignite.cache.store.CacheStore} which allows to store Ignite cache data into Cassandra tables. + */ +public class CassandraDirectPersistenceLoadTest extends LoadTestDriver { + /** */ + private static final Logger LOGGER = Logger.getLogger("CassandraLoadTests"); + + /** + * + * @param args Test arguments. + */ + public static void main(String[] args) { + try { + LOGGER.info("Cassandra load tests execution started"); + + LoadTestDriver driver = new CassandraDirectPersistenceLoadTest(); + + /** + * Load test scripts could be executed from several machines. Current implementation can correctly, + * handle situation when Cassandra keyspace/table was dropped - for example by the same load test + * started a bit later on another machine. Moreover there is a warm up period for each load test. + * Thus all the delays related to keyspaces/tables recreation actions will not affect performance metrics, + * but it will be produced lots of "trash" output in the logs (related to correct handling of such + * exceptional situation and keyspace/table recreation). + * + * Thus dropping test keyspaces at the beginning of the tests makes sense only for Unit tests, + * but not for Load tests. + **/ + + //CassandraHelper.dropTestKeyspaces(); + + driver.runTest("WRITE", WriteWorker.class, WriteWorker.LOGGER_NAME); + + driver.runTest("BULK_WRITE", BulkWriteWorker.class, BulkWriteWorker.LOGGER_NAME); + + driver.runTest("READ", ReadWorker.class, ReadWorker.LOGGER_NAME); + + driver.runTest("BULK_READ", BulkReadWorker.class, BulkReadWorker.LOGGER_NAME); + + /** + * Load test script executed on one machine could complete earlier that the same load test executed from + * another machine. Current implementation can correctly handle situation when Cassandra keyspace/table + * was dropped (simply recreate it). But dropping keyspace/table during load tests execution and subsequent + * recreation of such objects can have SIGNIFICANT EFFECT on final performance metrics. + * + * Thus dropping test keyspaces at the end of the tests makes sense only for Unit tests, + * but not for Load tests. + */ + + //CassandraHelper.dropTestKeyspaces(); // REVIEW This line is commented by purpose? + + LOGGER.info("Cassandra load tests execution completed"); + } + catch (Throwable e) { + LOGGER.error("Cassandra load tests execution failed", e); + throw new RuntimeException("Cassandra load tests execution failed", e); + } + finally { + CassandraHelper.releaseCassandraResources(); + } + } + + /** {@inheritDoc} */ + @Override protected Logger logger() { + return LOGGER; + } + + /** {@inheritDoc} */ + @Override protected Object setup(String logName) { + return CacheStoreHelper.createCacheStore( + TestsHelper.getLoadTestsCacheName(), + TestsHelper.getLoadTestsPersistenceSettings(), + CassandraHelper.getAdminDataSrc(), + Logger.getLogger(logName)); + } + +}
