Repository: incubator-blur Updated Branches: refs/heads/master 8e97ac237 -> 289de4e9e
Adding a basic jmeter plugin that can perform a load test on Blur. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/5fb575ac Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/5fb575ac Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/5fb575ac Branch: refs/heads/master Commit: 5fb575acbcd9e962a98d6de3abc9cc986a44bd2c Parents: 8b7eb8f Author: Aaron McCurry <amccu...@gmail.com> Authored: Wed Nov 11 20:19:25 2015 -0500 Committer: Aaron McCurry <amccu...@gmail.com> Committed: Wed Nov 11 20:19:25 2015 -0500 ---------------------------------------------------------------------- blur-benchmark/pom.xml | 149 +++++++++++ .../apache/blur/jmeter/BlurSamplerClient.java | 260 +++++++++++++++++++ 2 files changed, 409 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fb575ac/blur-benchmark/pom.xml ---------------------------------------------------------------------- diff --git a/blur-benchmark/pom.xml b/blur-benchmark/pom.xml new file mode 100644 index 0000000..625397e --- /dev/null +++ b/blur-benchmark/pom.xml @@ -0,0 +1,149 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor + license agreements. See the NOTICE file distributed with this work for additional + information regarding copyright ownership. The ASF licenses this file to + you under the Apache License, Version 2.0 (the "License"); you may not use + this file except in compliance with the License. You may obtain a copy of + the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required + by applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied. See the License for the specific + language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.blur</groupId> + <artifactId>blur</artifactId> + <version>0.3.0.incubating</version> + <relativePath>../pom.xml</relativePath> + </parent> + <groupId>org.apache.blur</groupId> + <artifactId>blur-benchmark</artifactId> + <version>${projectVersion}</version> + <packaging>jar</packaging> + <name>Blur Benchmark</name> + + <dependencies> + <dependency> + <groupId>org.apache.jmeter</groupId> + <artifactId>ApacheJMeter_java</artifactId> + <version>${jmeter.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>commons-math3</groupId> + <artifactId>commons-math3</artifactId> + </exclusion> + <exclusion> + <groupId>commons-pool2</groupId> + <artifactId>commons-pool2</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.blur</groupId> + <artifactId>blur-thrift</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + + <repositories> + <repository> + <id>libdir</id> + <url>file://${basedir}/../lib</url> + </repository> + </repositories> + + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <executions> + <execution> + <id>attach-sources</id> + <goals> + <goal>jar</goal> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-help-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>generate-resources</phase> + <goals> + <goal>effective-pom</goal> + </goals> + <configuration> + <output>${project.build.directory}/effective-pom.xml</output> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <version>2.3.1</version> + <executions> + <execution> + <phase>install</phase> + <goals> + <goal>install-file</goal> + </goals> + <configuration> + <file>${project.build.directory}/${artifactId}-${project.version}.jar</file> + <pomFile>${project.build.directory}/effective-pom.xml</pomFile> + <!-- sources></sources --> + <!-- javadoc></javadoc --> + <groupId>${project.groupId}</groupId> + <artifactId>${project.artifactId}</artifactId> + <version>${project.version}</version> + <packaging>jar</packaging> + <!--classifier></classifier --> + <generatePom>true</generatePom> + <createChecksum>true</createChecksum> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/5fb575ac/blur-benchmark/src/main/java/org/apache/blur/jmeter/BlurSamplerClient.java ---------------------------------------------------------------------- diff --git a/blur-benchmark/src/main/java/org/apache/blur/jmeter/BlurSamplerClient.java b/blur-benchmark/src/main/java/org/apache/blur/jmeter/BlurSamplerClient.java new file mode 100644 index 0000000..eff7353 --- /dev/null +++ b/blur-benchmark/src/main/java/org/apache/blur/jmeter/BlurSamplerClient.java @@ -0,0 +1,260 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.blur.jmeter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; + +import org.apache.blur.thirdparty.thrift_0_9_0.TException; +import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TBinaryProtocol; +import org.apache.blur.thirdparty.thrift_0_9_0.protocol.TProtocol; +import org.apache.blur.thirdparty.thrift_0_9_0.transport.TMemoryBuffer; +import org.apache.blur.thrift.BlurClient; +import org.apache.blur.thrift.generated.Blur.Iface; +import org.apache.blur.thrift.generated.BlurQuery; +import org.apache.blur.thrift.generated.BlurResult; +import org.apache.blur.thrift.generated.BlurResults; +import org.apache.blur.thrift.generated.Column; +import org.apache.blur.thrift.generated.FetchRecordResult; +import org.apache.blur.thrift.generated.FetchResult; +import org.apache.blur.thrift.generated.FetchRowResult; +import org.apache.blur.thrift.generated.Query; +import org.apache.blur.thrift.generated.Record; +import org.apache.blur.thrift.generated.Row; +import org.apache.blur.thrift.generated.Selector; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.jmeter.config.Arguments; +import org.apache.jmeter.protocol.java.sampler.AbstractJavaSamplerClient; +import org.apache.jmeter.protocol.java.sampler.JavaSamplerContext; +import org.apache.jmeter.samplers.SampleResult; + +public class BlurSamplerClient extends AbstractJavaSamplerClient implements Serializable { + + private static final Log LOG = LogFactory.getLog(BlurSamplerClient.class); + + private static List<Entry<String, String>> _fieldValueCache = new ArrayList<>(); + private static final long serialVersionUID = 1L; + private String _table; + private Iface _client; + private int _fieldValueCacheSize = 10000; + private ThreadLocal<Random> _random = new ThreadLocal<Random>() { + @Override + protected Random initialValue() { + return new Random(); + } + }; + + @Override + public Arguments getDefaultParameters() { + Arguments arguments = new Arguments(); + arguments.addArgument("ZooKeeperConnection", "localhost", + "The ZooKeeper connection string to the Blur cluster you want to test."); + arguments.addArgument("Table", "test"); + return arguments; + } + + @Override + public void setupTest(JavaSamplerContext context) { + String zkConnectionString = context.getParameter("ZooKeeperConnection"); + _table = context.getParameter("Table"); + _client = BlurClient.getClientFromZooKeeperConnectionStr(zkConnectionString); + } + + @Override + public SampleResult runTest(JavaSamplerContext context) { + SampleResult sampleResult = new SampleResult(); + BlurResults blurResults = null; + try { + BlurQuery blurQuery = getBlurQuery(); + sampleResult.sampleStart(); + blurResults = _client.query(_table, blurQuery); + sampleResult.sampleEnd(); + int size = getBytes(blurResults); + sampleResult.setBytes(size); + sampleResult.setSuccessful(true); + sampleResult.setResponseOK(); + } catch (Throwable t) { + sampleResult.setResponseMessage("Exception " + t.getMessage()); + sampleResult.setSuccessful(false); + LOG.error("Unknown error.", t); + } finally { + processResults(blurResults); + } + return sampleResult; + } + + private int getBytes(BlurResults blurResults) { + TMemoryBuffer trans = new TMemoryBuffer(1024); + TProtocol oprot = new TBinaryProtocol(trans); + try { + blurResults.write(oprot); + } catch (TException t) { + LOG.error("Unknown error.", t); + } + return trans.length(); + } + + private BlurQuery getBlurQuery() { + BlurQuery blurQuery = new BlurQuery(); + blurQuery.setSelector(new Selector()); + Query query = new Query(); + query.setQuery(getQueryString()); + blurQuery.setQuery(query); + return blurQuery; + } + + private String getQueryString() { + Random random = getRandom(); + int numberOfClauses = random.nextInt(9) + 1; + StringBuilder builder = new StringBuilder(); + int size = _fieldValueCache.size(); + if (size == 0) { + return "*"; + } + for (int i = 0; i < numberOfClauses; i++) { + int index = random.nextInt(size); + Entry<String, String> entry = _fieldValueCache.get(index); + builder.append("<" + entry.getKey() + ":" + entry.getValue() + "> "); + } + String query = builder.toString(); + // LOG.info("Query [" + query + "]"); + return query; + } + + private void processResults(BlurResults blurResults) { + if (blurResults == null) { + return; + } + for (BlurResult blurResult : blurResults.getResults()) { + processResult(blurResult); + } + } + + private void processResult(BlurResult blurResult) { + if (blurResult == null) { + return; + } + processResult(blurResult.getFetchResult()); + } + + private void processResult(FetchResult fetchResult) { + if (fetchResult == null) { + return; + } + processResult(fetchResult.getRowResult()); + processResult(fetchResult.getRecordResult()); + } + + private void processResult(FetchRecordResult recordResult) { + if (recordResult == null) { + return; + } + addRowId(recordResult.getRowid()); + processResult(recordResult.getRecord()); + } + + private void processResult(FetchRowResult rowResult) { + if (rowResult == null) { + return; + } + processResult(rowResult.getRow()); + } + + private void processResult(Row row) { + if (row == null) { + return; + } + addRowId(row.getId()); + processResult(row.getRecords()); + } + + private void processResult(List<Record> records) { + if (records == null) { + return; + } + for (Record record : records) { + processResult(record); + } + } + + private void addRowId(String id) { + addFieldToQuery("rowid", id); + } + + private void processResult(Record record) { + if (record == null) { + return; + } + addRecordId(record.getRecordId()); + List<Column> columns = record.getColumns(); + if (columns != null) { + for (Column column : columns) { + addFieldToQuery(record.getFamily() + "." + column.getName(), column.getValue()); + } + } + } + + private void addRecordId(String id) { + addFieldToQuery("recordid", id); + } + + private void addFieldToQuery(String fieldName, String value) { + if (StringUtils.isEmpty(fieldName) || StringUtils.isEmpty(value)) { + return; + } + Entry<String, String> e = toEntry(fieldName, value); + // LOG.info("Adding [" + fieldName + "] [" + value + "]"); + synchronized (_fieldValueCache) { + if (_fieldValueCache.size() >= _fieldValueCacheSize) { + Random random = getRandom(); + int index = random.nextInt(_fieldValueCache.size()); + _fieldValueCache.set(index, e); + } else { + _fieldValueCache.add(e); + } + } + } + + private Random getRandom() { + return _random.get(); + } + + private static Entry<String, String> toEntry(String fieldName, String value) { + return new Entry<String, String>() { + + @Override + public String setValue(String value) { + throw new RuntimeException("Not implemented."); + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getKey() { + return fieldName; + } + }; + } +}
