Add insert-loadgen example to repo This is a load generator designed to write random data to an arbitrary schema as quickly as possible.
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/565743a4 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/565743a4 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/565743a4 Branch: refs/heads/master Commit: 565743a4ca4ed52af25b3b7275de1a1e3467d1f3 Parents: 4de3201 Author: Mike Percy <mpe...@cloudera.com> Authored: Thu Nov 5 14:29:03 2015 -0800 Committer: Mike Percy <mpe...@cloudera.com> Committed: Fri Nov 6 16:46:12 2015 -0800 ---------------------------------------------------------------------- java/.gitignore | 2 + java/insert-loadgen/README | 17 +++ java/insert-loadgen/pom.xml | 72 ++++++++++ .../kududb/examples/loadgen/InsertLoadgen.java | 140 +++++++++++++++++++ 4 files changed, 231 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/565743a4/java/.gitignore ---------------------------------------------------------------------- diff --git a/java/.gitignore b/java/.gitignore index 4afb9e4..b2dce6e 100644 --- a/java/.gitignore +++ b/java/.gitignore @@ -3,3 +3,5 @@ target/ .classpath .project .settings +.idea +*.iml http://git-wip-us.apache.org/repos/asf/kudu/blob/565743a4/java/insert-loadgen/README ---------------------------------------------------------------------- diff --git a/java/insert-loadgen/README b/java/insert-loadgen/README new file mode 100644 index 0000000..7c0628e --- /dev/null +++ b/java/insert-loadgen/README @@ -0,0 +1,17 @@ +Random insert load generator. This will insert as fast as it can using +AUTO_BACKGROUND_FLUSH with the specified number of threads. All fields are +randomized, and insert failures (including errors from collisions) are ignored. + +To build and run, do the following: + +$ mvn package +$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar kudu_master_host kudu_table_name num_threads + +For example, if you are running the Quickstart VM with the host name +"quickstart.cloudera", then you can use: + +$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar quickstart.cloudera test_table 16 + +Note: This program will not create the "test_table" table. You must do that +via other means, such as through impala-shell or using the create-demo-table +program included in the Kudu source tree. http://git-wip-us.apache.org/repos/asf/kudu/blob/565743a4/java/insert-loadgen/pom.xml ---------------------------------------------------------------------- diff --git a/java/insert-loadgen/pom.xml b/java/insert-loadgen/pom.xml new file mode 100644 index 0000000..13f5cd5 --- /dev/null +++ b/java/insert-loadgen/pom.xml @@ -0,0 +1,72 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <groupId>kudu-examples</groupId> + <artifactId>kudu-insert-loadgen</artifactId> + <packaging>jar</packaging> + <version>0.1-SNAPSHOT</version> + <name>Random Insert Load Generator for Kudu</name> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>2.3.1</version> + <configuration> + <source>1.7</source> + <target>1.7</target> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.4</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.kududb.examples.loadgen.InsertLoadgen</mainClass> + </transformer> + </transformers> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <repositories> + <repository> + <id>cdh.repo</id> + <name>Cloudera Repositories</name> + <url>https://repository.cloudera.com/artifactory/cloudera-repos</url> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + </repositories> + + <dependencies> + + <dependency> + <groupId>org.kududb</groupId> + <artifactId>kudu-client</artifactId> + <version>0.5.0</version> + </dependency> + + <!-- for logging messages --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-simple</artifactId> + <version>1.7.12</version> + </dependency> + + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/kudu/blob/565743a4/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java ---------------------------------------------------------------------- diff --git a/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java new file mode 100644 index 0000000..f7438c0 --- /dev/null +++ b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java @@ -0,0 +1,140 @@ +package org.kududb.examples.loadgen; + +import org.kududb.Schema; +import org.kududb.Type; +import org.kududb.client.Insert; +import org.kududb.client.KuduClient; +import org.kududb.client.KuduSession; +import org.kududb.client.KuduTable; +import org.kududb.client.PartialRow; +import org.kududb.client.SessionConfiguration; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; + +public class InsertLoadgen { + + private static class RandomDataGenerator { + private final Random rng; + private final int index; + private final Type type; + + /** + * Instantiate a random data generator for a specific field. + * @param index The numerical index of the column in the row schema + * @param type The type of the data at index {@code index} + */ + public RandomDataGenerator(int index, Type type) { + this.rng = new Random(); + this.index = index; + this.type = type; + } + + /** + * Add random data to the given row for the column at index {@code index} + * of type {@code type} + * @param row The row to add the field to + */ + void generateColumnData(PartialRow row) { + switch (type) { + case INT8: + row.addByte(index, (byte) rng.nextInt(Byte.MAX_VALUE)); + return; + case INT16: + row.addShort(index, (short)rng.nextInt(Short.MAX_VALUE)); + return; + case INT32: + row.addInt(index, rng.nextInt(Integer.MAX_VALUE)); + return; + case INT64: + case TIMESTAMP: + row.addLong(index, rng.nextLong()); + return; + case BINARY: + byte bytes[] = new byte[16]; + rng.nextBytes(bytes); + row.addBinary(index, bytes); + return; + case STRING: + row.addString(index, UUID.randomUUID().toString()); + return; + case BOOL: + row.addBoolean(index, rng.nextBoolean()); + return; + case FLOAT: + row.addFloat(index, rng.nextFloat()); + return; + case DOUBLE: + row.addDouble(index, rng.nextDouble()); + return; + default: + throw new UnsupportedOperationException("Unknown type " + type); + } + } + } + + public static void runLoad(String masterHost, String tableName) { + KuduClient client = new KuduClient.KuduClientBuilder(masterHost).build(); + + try { + KuduTable table = client.openTable(tableName); + Schema schema = table.getSchema(); + List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount()); + for (int i = 0; i < schema.getColumnCount(); i++) { + generators.add(new RandomDataGenerator(i, schema.getColumnByIndex(i).getType())); + } + + KuduSession session = client.newSession(); + session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); + while (true) { + Insert insert = table.newInsert(); + PartialRow row = insert.getRow(); + for (int i = 0; i < schema.getColumnCount(); i++) { + generators.get(i).generateColumnData(row); + } + session.apply(insert); + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + client.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + public static void main(String[] args) { + if (args.length != 3) { + System.err.println("Usage: InsertLoadgen kudu_master_host kudu_table num_threads"); + System.exit(1); + } + + final String masterHost = args[0]; + final String tableName = args[1]; + int numThreads = Integer.parseInt(args[2]); + + final CountDownLatch latch = new CountDownLatch(numThreads); + + List<Thread> threads = new ArrayList<>(numThreads); + for (int i = 0; i < numThreads; i++) { + threads.add(new Thread(new Runnable() { + public void run() { + runLoad(masterHost, tableName); + latch.countDown(); + } + })); + threads.get(i).start(); + } + try { + latch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +}
