http://git-wip-us.apache.org/repos/asf/usergrid/blob/29bf682e/tests/performance/runRandomEntityByNameTest.sh ---------------------------------------------------------------------- diff --git a/tests/performance/runRandomEntityByNameTest.sh b/tests/performance/runRandomEntityByNameTest.sh new file mode 100755 index 0000000..f52747b --- /dev/null +++ b/tests/performance/runRandomEntityByNameTest.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +. "$DIR/testConfig.sh" + +# from testConfig.sh +#URL= +#ADMIN_USER= +#ADMIN_PASSWORD= +#ORG= +#APP= +#AUTH_TYPE= +#TOKEN_TYPE= +#CREATE_ORG= +#CREATE_APP= +#LOAD_ENTITIES= +#SANDBOX_COLLECTION= +#NUM_ENTITIES= #may be overridden on command line +#SKIP_SETUP= +#COLLECTION= +#ENTITY_PREFIX= +#ENTITY_SEED= #may be overridden on command line +#RETRY_COUNT= +#END_CONDITION_TYPE= +#END_MINUTES= +#END_REQUEST_COUNT= +#CONSTANT_USERS_PER_SEC= +#CONSTANT_USERS_DURATION= + +die() { echo "$@" 1>&2 ; exit 1; } + +[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [QUERY_PARAMS]]]" + +RAMP_USERS="$1" +RAMP_TIME="$2" +[ "$#" -ge 3 ] && NUM_ENTITIES="$3" +[ "$#" -ge 4 ] && ENTITY_SEED="$4" +[ "$#" -ge 5 ] && QUERY_PARAMS="$5" + +shift $# + +SCENARIO_TYPE=nameRandomInfinite + +#Compile everything +mvn compile + +#Execute the test +mvn gatling:execute \ +-DbaseUrl=${URL} \ +-DadminUser=${ADMIN_USER} \ +-DadminPassword=${ADMIN_PASSWORD} \ +-Dorg=${ORG} \ +-Dapp=${APP} \ +-DauthType=${AUTH_TYPE} \ +-DtokenType=${TOKEN_TYPE} \ +-DcreateOrg=${CREATE_ORG} \ +-DcreateApp=${CREATE_APP} \ +-DloadEntities=${LOAD_ENTITIES} \ +-DsandboxCollection=${SANDBOX_COLLECTION} \ +-DnumEntities=${NUM_ENTITIES} \ +-DskipSetup=${SKIP_SETUP} \ +-Dcollection=${COLLECTION} \ +-DentityPrefix=${ENTITY_PREFIX} \ +-DentitySeed=${ENTITY_SEED} \ +-DretryCount=${RETRY_COUNT} \ +-DendConditionType=${END_CONDITION_TYPE} \ +-DendMinutes=${END_MINUTES} \ +-DendRequestCount=${END_REQUEST_COUNT} \ +-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \ +-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \ +-DscenarioType=${SCENARIO_TYPE} \ +-DrampUsers=${RAMP_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \ +-DqueryParams=${QUERY_PARAMS} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation +
http://git-wip-us.apache.org/repos/asf/usergrid/blob/29bf682e/tests/performance/runUpdateEntities.sh ---------------------------------------------------------------------- diff --git a/tests/performance/runUpdateEntities.sh b/tests/performance/runUpdateEntities.sh new file mode 100755 index 0000000..579d0c7 --- /dev/null +++ b/tests/performance/runUpdateEntities.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DIR="${BASH_SOURCE%/*}" +if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi +. "$DIR/testConfig.sh" + +# from testConfig.sh +#URL= +#ADMIN_USER= +#ADMIN_PASSWORD= +#ENTITY_WORKER_NUM= #may be overridden on command line +#ENTITY_WORKER_COUNT= #may be overridden on command line +#ORG= +#APP= +#AUTH_TYPE= +#TOKEN_TYPE= +#CREATE_ORG= +#CREATE_APP= +#LOAD_ENTITIES= +#SANDBOX_COLLECTION= +#NUM_ENTITIES= #may be overridden on command line +#SKIP_SETUP= +#COLLECTION= +#ENTITY_PREFIX= +#ENTITY_SEED= #may be overridden on command line +#RETRY_COUNT= +#CONSTANT_USERS_PER_SEC= +#CONSTANT_USERS_DURATION= + +die() { echo "$@" 1>&2 ; exit 1; } + +[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]" + +RAMP_USERS="$1" +RAMP_TIME="$2" +[ "$#" -ge 3 ] && NUM_ENTITIES="$3" +[ "$#" -ge 4 ] && ENTITY_SEED="$4" +[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5" +[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6" + +shift $# + +SCENARIO_TYPE=updateEntities +UPDATE_PROPERTY=updateProp + +#Compile everything +mvn compile + +#Execute the test +mvn gatling:execute \ +-DbaseUrl=${URL} \ +-DadminUser=${ADMIN_USER} \ +-DadminPassword=${ADMIN_PASSWORD} \ +-DentityWorkerNum=${ENTITY_WORKER_NUM} \ +-DentityWorkerCount=${ENTITY_WORKER_COUNT} \ +-Dorg=${ORG} \ +-Dapp=${APP} \ +-DauthType=${AUTH_TYPE} \ +-DtokenType=${TOKEN_TYPE} \ +-DcreateOrg=${CREATE_ORG} \ +-DcreateApp=${CREATE_APP} \ +-DloadEntities=${LOAD_ENTITIES} \ +-DsandboxCollection=${SANDBOX_COLLECTION} \ +-DnumEntities=${NUM_ENTITIES} \ +-DskipSetup=${SKIP_SETUP} \ +-Dcollection=${COLLECTION} \ +-DentityPrefix=${ENTITY_PREFIX} \ +-DentitySeed=${ENTITY_SEED} \ +-DretryCount=${RETRY_COUNT} \ +-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \ +-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \ +-DscenarioType=${SCENARIO_TYPE} \ +-DupdateProperty=${UPDATE_PROPERTY} \ +-DrampUsers=${RAMP_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation + http://git-wip-us.apache.org/repos/asf/usergrid/blob/29bf682e/tests/performance/runtests_deprecated.sh ---------------------------------------------------------------------- diff --git a/tests/performance/runtests_deprecated.sh b/tests/performance/runtests_deprecated.sh new file mode 100755 index 0000000..ea9658c --- /dev/null +++ b/tests/performance/runtests_deprecated.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +die() { echo "$@" 1>&2 ; exit 1; } + +#### +#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins +#Then aggregate the results +#### +[ "$#" -eq 4 ] || die "4 arguments required, $# provided. Example is $0 URL MAX_CONCURRENT_USERS RAMP_TIME(seconds) DURATION_TIME(seconds)" + +URL="$1" +MAX_CONCURRENT_USERS="$2" +RAMP_TIME="$3" +DURATION_TIME="$4" + +shift 4 + +#Compile everything +mvn compile + +#Set the app id to be a date epoch for uniqueness +APP1=$(date +%s) + + + +#Execute the post step +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.PostUsersSimulation \ +-Dapp=${APP1} + + +#Execute the get users by username +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.GetUsersSimulation \ +-Dapp=${APP1} + + +#Execute the get users by page +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.GetUserPagesSimulation \ +-Dapp=${APP1} + + +APP2=$(date +%s) + +#Execute put users to create them +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.PutUsersSimulation \ +-Dapp=${APP2} + +#Execute the put users to update them +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-rampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.PutUsersSimulation \ +-Dapp=${APP2} + + +#Execute the delete to remove them +mvn gatling:execute -Dorg=usergrid \ +-Dbaseurl=${URL} \ +-DmaxPossibleUsers=${MAX_CONCURRENT_USERS} \ +-DrampTime=${RAMP_TIME} \ +-DadminUser=usergrid \ +-DadminPassword=test \ +-Dduration=${DURATION_TIME} \ +-Dgatling.simulationClass=org.apache.usergrid.simulations.DeleteUsersSimulation \ +-Dapp=${APP2} + + +#Now move all the reports +#AGGREGATE_DIR="target/aggregate-$(date +%s)" + +#mkdir -p ${AGGREGATE_DIR} + + +#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log +#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \; + http://git-wip-us.apache.org/repos/asf/usergrid/blob/29bf682e/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala ---------------------------------------------------------------------- diff --git a/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala new file mode 100755 index 0000000..326a319 --- /dev/null +++ b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.datagenerators + +import org.apache.usergrid.enums.EntityType +import org.apache.usergrid.helpers.Utils +import org.apache.usergrid.settings.Settings + +import scala.collection.mutable.ArrayBuffer +import scala.util.parsing.json.JSONObject + +object EntityDataGenerator { + + def generateBlockUserLists(numUsers: Int): Map[String, String] = { + + var blocks: ArrayBuffer[String] = new ArrayBuffer[String] + var blockedBy: ArrayBuffer[String] = new ArrayBuffer[String] + + for (numBlock <- 1 to Utils.generateRandomInt(1, 7)) { + blocks += "user".concat(Utils.generateRandomInt(0, numUsers).toString) + } + + for (numBlockedBy <- 1 to Utils.generateRandomInt(1, 7)) { + blockedBy += "user".concat(Utils.generateRandomInt(0, numUsers).toString) + } + + return Map("blocks" -> blocks.toArray.mkString(","), "blockedBy" -> blockedBy.toArray.mkString(",")) + + } + + def generateUser(userId: String): Map[String,String] = { + + Map( + + "username" -> "user".concat(userId), + "profileId" -> Utils.generateRandomInt(10000, 1000000).toString, + "displayName" -> Utils.generateRandomInt(10000, 1000000).toString, + "showAge" -> Utils.generateRandomInt(0, 1).toString, + "ethnicity" -> Utils.generateRandomInt(1, 15).toString, + "relationshipStatus" -> Utils.generateRandomInt(1, 4).toString, + "headline" -> "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "aboutMe" -> "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "age" -> Utils.generateRandomInt(18, 65).toString, + "height" -> Utils.generateRandomInt(48, 84).toString, + "weight" -> Utils.generateRandomInt(120, 350).toString, + "seen" -> Utils.generateRandomInt(50, 100000).toString, + "password" -> "password" + ) + } + + def generateCustomEntity(): Map[String,String] = { + + val entity: Map[String, String] = Map( + // "name" -> "fdsa", + "address" -> Utils.generateRandomInt(10000, 1000000).toString, + "city" -> Utils.generateRandomInt(10000, 1000000).toString, + "state" -> Utils.generateRandomInt(10000, 1000000).toString, + "zip" -> Utils.generateRandomInt(10000, 1000000).toString, + "phone" -> Utils.generateRandomInt(10000, 1000000).toString, + "businessname" -> Utils.generateRandomInt(0, 1).toString, + "menu" -> Utils.generateRandomInt(1, 1000000).toString, + "specials" -> Utils.generateRandomInt(1, 1000000).toString, + "profile" -> "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "description" -> "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "directions" -> Utils.generateRandomInt(18, 65).toString, + "atmosphere" -> Utils.generateRandomInt(48, 84).toString, + "bar" -> Utils.generateRandomInt(120, 350).toString, + "tables" -> Utils.generateRandomInt(50, 100000).toString, + "outdoor" -> Utils.generateRandomInt(50, 100000).toString + ) + + Map("entity" -> new JSONObject(entity).toString()) + } + + /* --------------------------- */ + + def generateTrivialEntity(name: String = null): String = { + + val nameKey = if (name != null) "name" else "noname" + val nameVal = if (name != null) name else Utils.generateRandomInt(1,10000000).toString + + val entity: Map[String, String] = Map( + nameKey -> nameVal + ) + + new JSONObject(entity).toString() + } + + def generateTrivialSortableEntity(name: String = null): String = { + + if (name != null) + new JSONObject(Map("name" -> name, "sortField" -> Utils.generateRandomInt(1,10000000))).toString() + else + new JSONObject(Map("sortField" -> Utils.generateRandomInt(1,10000000))).toString() + } + + val pirate1KString = "Yarrrrrrrrrrrrrrrr! Blimey! Avast ye scurvy dog! Dead men tell no tales. Hang him from the yardarm! Heave ho! Run a shot across the bow. Yo-ho-ho and a bottle of rum. Heave ho ye scallywag, or ye shall walk the plank! Weigh anchor and hoist the mizzen! Thar she blows! Shiver me timbers! Splice the mainbrace! Keelhaul the hornswaggler! Raise the Jolly Roger! Feed him to the fish! You fight like a dairy farmer. How appropriate, you fight like a cow. Batten down the hatches! Blow the man down! Swab the deck! Ahoy, matey! I'll crush ye barnacles! Fetch me grog! Gangway! Arrr, he's gone to Davy Jones's Locker. He be three sheets to the wind. Yo-ho me hearties! Prepare to be boarded! All your pieces of eight, else ye meet me cat o'nine tails! Bring 'er alongside! The rougher the seas, the smoother we sail! It's more fun to be a pirate than to join the navy. The beatings will continue until morale improves! Fifteen men on the dead man's chest! I be the captain of th is ship -- swab the decks! " + + def generateLargeMultiFieldEntity(name: String = null, entityNum: Int = 0): String = { + + val objectMap:scala.collection.mutable.HashMap[String,Any] = scala.collection.mutable.HashMap.empty[String,Any] + if (name != null) objectMap += ("name" -> name) + if (Settings.entityNumberProperty != null && Settings.entityNumberProperty != "") objectMap += (Settings.entityNumberProperty -> entityNum) + val fieldValue = pirate1KString * Settings.multiPropertySizeInK + for (i <- 1 to Settings.multiPropertyCount) { + objectMap += (s"${Settings.multiPropertyPrefix}$i" -> fieldValue) + } + + new JSONObject(objectMap.toMap).toString() + } + + def generateBasicEntity(name: String = null): String = { + + val nameKey = if (name != null) "name" else "noname" + val nameVal = if (name != null) name else Utils.generateRandomInt(1,10000000).toString + + val entity: Map[String, String] = Map( + nameKey -> nameVal, + "address" -> Utils.generateRandomInt(10000, 1000000).toString, + "city" -> Utils.generateRandomInt(10000, 1000000).toString, + "state" -> Utils.generateRandomInt(10000, 1000000).toString, + "zip" -> Utils.generateRandomInt(10000, 1000000).toString, + "phone" -> Utils.generateRandomInt(10000, 1000000).toString, + "businessname" -> Utils.generateRandomInt(0, 1).toString, + "menu" -> Utils.generateRandomInt(1, 1000000).toString, + "specials" -> Utils.generateRandomInt(1, 1000000).toString, + "profile" -> "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "description" -> "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "directions" -> Utils.generateRandomInt(18, 65).toString, + "atmosphere" -> Utils.generateRandomInt(48, 84).toString, + "bar" -> Utils.generateRandomInt(120, 350).toString, + "tables" -> Utils.generateRandomInt(50, 100000).toString, + "outdoor" -> Utils.generateRandomInt(50, 100000).toString + ) + + new JSONObject(entity).toString() + } + + def generateEntity(entityType: String = EntityType.Basic, entityName: String = null, entityNum: Int = 0): String = { + + entityType match { + case EntityType.Trivial => generateTrivialEntity(entityName) + case EntityType.TrivialSortable => generateTrivialSortableEntity(entityName) + case EntityType.Basic => generateBasicEntity(entityName) + case EntityType.LargeMultiField => generateLargeMultiFieldEntity(entityName, entityNum) + } + } + + def generateUserEntity(userId: String = "user" + System.currentTimeMillis().toString): String = { + + val entity: Map[String, Any] = Map( + "username" -> userId, + "profileId" -> Utils.generateRandomInt(10000, 1000000), + "displayName" -> Utils.generateRandomInt(10000, 1000000).toString, + "showAge" -> Utils.generateRandomInt(0, 1), + "ethnicity" -> Utils.generateRandomInt(1, 15), + "relationshipStatus" -> Utils.generateRandomInt(1, 4), + "headline" -> "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "aboutMe" -> "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "age" -> Utils.generateRandomInt(18, 65), + "height" -> Utils.generateRandomInt(48, 84), + "weight" -> Utils.generateRandomInt(120, 350), + "seen" -> Utils.generateRandomInt(50, 100000).toString, + "password" -> "password" + ) + + new JSONObject(entity).toString() + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/29bf682e/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala ---------------------------------------------------------------------- diff --git a/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala new file mode 100755 index 0000000..a9225b2 --- /dev/null +++ b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.datagenerators + +import java.util.UUID +import java.util.concurrent.atomic.AtomicInteger +import io.gatling.core.Predef._ +import org.apache.usergrid.helpers.{Setup, Utils} +import org.apache.usergrid.settings.Settings +import scala.collection.mutable.ArrayBuffer +import scala.io.Source + +object FeederGenerator { + + def generateUserWithGeolocationFeeder(numUsers: Int, radius: Double, centerLatitude: Double, centerLongitude: Double): Array[Map[String, String]] = { + var userArray: ArrayBuffer[Map[String, String]] = new ArrayBuffer[Map[String, String]] + for (userCount <- 1 to numUsers) { + var user: Map[String, String] = EntityDataGenerator.generateUser(userCount.toString) + val geolocation: Map[String, String] = Utils.generateRandomGeolocation(radius, centerLatitude, centerLongitude) + val blockLists: Map[String, String] = EntityDataGenerator.generateBlockUserLists(numUsers) + + user = user ++ geolocation ++ blockLists + + userArray += user + } + + userArray.toArray + } + + /** + * Generate users forever + * @param seed + * @param radius + * @param centerLatitude + * @param centerLongitude + * @return + */ + def generateUserWithGeolocationFeederInfinite(seed:Int,radius: Double, centerLatitude: Double, centerLongitude: Double): Iterator[Map[String, String]] = { + Iterator.from(seed).map(i=>generateUserData(i.toString, radius, centerLatitude, centerLongitude)) + } + + /** + * Generate user data based on atomically increasing integers + * @param radius + * @param centerLatitude + * @param centerLongitude + * @return + */ + def generateUserData(id: String, radius: Double, centerLatitude: Double, centerLongitude: Double): Map[String, String] = { + + val user: Map[String, String] = EntityDataGenerator.generateUser(id) + val geolocation: Map[String, String] = Utils.generateRandomGeolocation(radius, centerLatitude, centerLongitude) + val blockLists: Map[String, String] = EntityDataGenerator.generateBlockUserLists(1) + + // merge maps + user ++ geolocation ++ blockLists + } + + + def generateGeolocationFeeder(radius: Double, centerLatitude: Double, centerLongitude: Double): Feeder[String] = { + + val geolocationFeeder = new Feeder[String] { + + // always return true as this feeder can be polled infinitively + override def hasNext = true + + override def next(): Map[String, String] = { + val geolocation: Map[String, String] = Utils.generateRandomGeolocation(radius, centerLatitude, centerLongitude) + Map("latitude" -> geolocation("latitude"), "longitude" -> geolocation("longitude")) + } + } + + geolocationFeeder + } + + def generateGeolocationWithQueryFeeder(radius: Double, centerLatitude: Double, centerLongitude: Double): Feeder[String] = { + + val geolocationFeeder = new Feeder[String] { + + // always return true as this feeder can be polled infinitively + override def hasNext = true + + override def next(): Map[String, String] = { + val geolocation: Map[String, String] = Utils.generateRandomGeolocation(radius, centerLatitude, centerLongitude) + val queryParams = Utils.generateRandomQueryString + Map("latitude" -> geolocation("latitude"), "longitude" -> geolocation("longitude"), "queryParams" -> queryParams) + } + } + + geolocationFeeder + } + + def generateUserConnectionFeeder(numUsers: Int): Feeder[String] = { + + val userIdFeeder = new Feeder[String] { + + // always return true as this feeder can be polled infinitely + override def hasNext = true + + override def next(): Map[String, String] = { + Map("user1" -> "user".concat(Utils.generateRandomInt(1, numUsers).toString), "user2" -> "user".concat(Utils.generateRandomInt(1, numUsers).toString)) + } + } + + userIdFeeder + } + + def generateEntityNameFeeder(prefix: String, numEntities: Int): Iterator[Map[String, String]] = { + Iterator.from(1).map(i=> Map("entityName" -> prefix.concat(i.toString).concat(UUID.randomUUID().toString))) + } + + def generateRandomEntityNameFeeder(prefix: String, numEntities: Int): Array[Map[String, String]] = { + + var nameArray: ArrayBuffer[Map[String, String]] = new ArrayBuffer[Map[String, String]] + + for (_ <- 1 to numEntities) { + nameArray += Map("entityName" -> prefix.concat(Utils.generateRandomInt(0, 100000000).toString)) + } + + nameArray.toArray + } + + + + + /** + * Generate users forever + * @param seed The seed + * @return + */ + def generateCustomEntityInfinite(seed:Int): Iterator[Map[String, String]] = { + //val rod = "rod" + val userFeeder = Iterator.from(seed).map(i=>EntityDataGenerator.generateCustomEntity()) + return userFeeder + } + + + + /** + * Generate users forever + * @param seed The seed + * @return + */ + def generateCustomEntityPutInfinite(seed:Int): Iterator[Map[String, Any]] = { + //val rod = "rod" + val userFeeder = Iterator.from(seed).map(i=>Map("entityName" -> i.toString.concat(UUID.randomUUID().toString), "entity" -> EntityDataGenerator.generateCustomEntity())); + return userFeeder + } + + + /* + def testFeeder(seed:Int): Iterator[Map[String, String]] = { + var entity: Map[String, String] = EntityDataGenerator.generateCustomEntity(); + Map("entity" -> entity) + val userFeeder = Iterator.from(seed).map(i=>EntityDataGenerator.generateCustomEntity()) + return userFeeder + } + + + def testFeeder(): Array[Map[String, String]] = { + var userArray: ArrayBuffer[Map[String, String]] = new ArrayBuffer[Map[String, String]] + for (userCount <- 1 to numUsers) { + var user: Map[String, String] = EntityDataGenerator.generateUser(userCount.toString) + var geolocation: Map[String, String] = Utils.generateRandomGeolocation(radius, centerLatitude, centerLongitude) + var blockLists: Map[String, String] = EntityDataGenerator.generateBlockUserLists(numUsers) + + user = user ++ geolocation ++ blockLists + + userArray += user + } + return userArray.toArray + } + */ + + /* --------------------------- */ + + /** + * Generate specified number of custom entities + * @param numEntities Number of entities to create + * @param entityType Type of entity to create + * @param prefix Prefix for entities + * @return + */ + def generateCustomEntityArray(numEntities: Int, entityType: String, prefix: String, seed: Int = 1): Array[String] = { + //val entityFeeder = Iterator.from(1).take(numEntities).map(i=>Map("entity" -> EntityDataGenerator.generateNamedCustomEntityJSONString(prefix.concat(i.toString())))) + var entityArray: ArrayBuffer[String] = new ArrayBuffer[String] + for (i <- seed to numEntities+seed-1) { + var entity = EntityDataGenerator.generateEntity(entityType, prefix.concat(i.toString), i) + entityArray += entity + } + + entityArray.toArray + } + + /* + * Gatling doesn't handle feeders running out of data well -- ending test with failure and not building a report. + * This feeder will serve data forever, but validEntity will be set to "no" when data has run out. Each user can + * then exit in a controlled fashion. + */ + def generateCustomEntityFeeder(numEntities: Int, entityType: String, prefix: String = "", seed: Int = 1): Feeder[String] = + new Feeder[String] { + var counter = new AtomicInteger(0) + + // runs forever -- users detect when data is done using validEntity field + override def hasNext: Boolean = true + + override def next(): Map[String, String] = { + val i = counter.getAndIncrement() + val seededVal = if (Settings.interleavedWorkerFeed) seed + (i * Settings.entityWorkerCount) + (Settings.entityWorkerNum - 1) else i + seed + val noPrefix = prefix == null || prefix == "" + val entityName = if (noPrefix) seededVal.toString else prefix.concat(seededVal.toString) + val entity = EntityDataGenerator.generateEntity(entityType, if (noPrefix) null else entityName, seededVal) + //println(entity) + val entityUrl = Settings.baseCollectionUrl + "/" + entityName + val validEntity = if (!Settings.unlimitedFeed && i >= numEntities) "no" else "yes" + val collectionName = Settings.app + "/" + Settings.collection + + // println(entityName) + + Map("entityName" -> entityName, "entity" -> entity, "entityUrl" -> entityUrl, "validEntity" -> validEntity, "entityNum" -> (i+1).toString, "seededEntityNum" -> seededVal.toString, + "collectionName" -> collectionName) + } + } + + def collectionNameFeeder: Feeder[String] = new Feeder[String] { + val list: List[String] = if (Settings.allApps) Setup.getApplicationCollectionsList else Setup.getCollectionsList() + var counter = new AtomicInteger(0) + + override def hasNext: Boolean = true + + override def next(): Map[String, String] = { + val i = counter.getAndIncrement() + val collectionName = if (i < list.length) list(i) else "" + val validEntity = if (i >= list.length) "no" else "yes" + + Map("collectionName" -> collectionName, "validEntity" -> validEntity) + } + } + + def collectionCsvFeeder: Feeder[String] = new Feeder[String] { + val csvLines = if (Settings.feedAuditUuids) Source.fromFile(Settings.feedAuditUuidFilename).getLines().toArray else Array[String]() + val csvLinesLen = csvLines.length + var counter = new AtomicInteger(0) + var entityCounter: Long = 0L + var lastEntityCountPrinted: Long = 0L + var entityProgressCount: Long = Settings.entityProgressCount + + override def hasNext: Boolean = true + + def getNextLine: String = { + do { + val i = counter.getAndIncrement() + if (i >= csvLinesLen) return null + + val line = csvLines(i) + if (line != Settings.uuidsHeader) return line + + } while (true) + + null + } + + override def next: Map[String, String] = { + val line = getNextLine + val validEntity = if (line == null) "no" else "yes" + val array = if (line != null) line.split(",") else null + val collectionName = if (line != null) array(0) else "" + val name = if (line != null) array(1) else "" + val uuid = if (line != null) array(2) else "" + val modified = if (line != null) array(3) else "" + val lastStatus = if (line != null) array(4) else "" + //println(s"$collectionName|$name|$uuid|$modified") + val accessField = if (uuid != "") uuid else name + val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'" + + if (entityProgressCount > 0L && entityCounter >= lastEntityCountPrinted + entityProgressCount) { + println(s"CSV Entity: $entityCounter") + lastEntityCountPrinted = entityCounter + } + + Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus, + "validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField) + } + } + + def generateCustomEntityInfiniteFeeder(seed: Int = Settings.entitySeed, entityType: String = Settings.entityType, prefix: String = Settings.entityPrefix): Iterator[String] = { + Iterator.from(seed).map(i=>EntityDataGenerator.generateEntity(entityType, if (prefix == null || prefix == "") null else prefix.concat(i.toString), i)) + } + +}
