[examples] Improve loadgen example This is a grab bag of improvements to the loadgen example: - Updated pom - README rewritten, in adoc format. - Small improvements to the Java code.
Change-Id: Ibc0e655f56adcedec2c3dd8a8b3c900e3b2e7803 Reviewed-on: http://gerrit.cloudera.org:8080/9853 Reviewed-by: Adar Dembo <a...@cloudera.com> Tested-by: Will Berkeley <wdberke...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/cc8c5e3e Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/cc8c5e3e Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/cc8c5e3e Branch: refs/heads/master Commit: cc8c5e3e8783c81f10233d531a361723fdceb3f9 Parents: 0804136 Author: Will Berkeley <wdberke...@apache.org> Authored: Wed Mar 28 21:56:08 2018 -0700 Committer: Will Berkeley <wdberke...@gmail.com> Committed: Tue Apr 3 04:29:44 2018 +0000 ---------------------------------------------------------------------- examples/java/insert-loadgen/README | 17 --- examples/java/insert-loadgen/README.adoc | 45 +++++++ examples/java/insert-loadgen/pom.xml | 49 ++++--- .../org/apache/kudu/examples/InsertLoadgen.java | 132 +++++++++++++++++++ .../kududb/examples/loadgen/InsertLoadgen.java | 110 ---------------- 5 files changed, 205 insertions(+), 148 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/README ---------------------------------------------------------------------- diff --git a/examples/java/insert-loadgen/README b/examples/java/insert-loadgen/README deleted file mode 100644 index 7c0628e..0000000 --- a/examples/java/insert-loadgen/README +++ /dev/null @@ -1,17 +0,0 @@ -Random insert load generator. This will insert as fast as it can using -AUTO_BACKGROUND_FLUSH with the specified number of threads. All fields are -randomized, and insert failures (including errors from collisions) are ignored. - -To build and run, do the following: - -$ mvn package -$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar kudu_master_host kudu_table_name num_threads - -For example, if you are running the Quickstart VM with the host name -"quickstart.cloudera", then you can use: - -$ java -jar target/kudu-insert-loadgen-0.1-SNAPSHOT.jar quickstart.cloudera test_table 16 - -Note: This program will not create the "test_table" table. You must do that -via other means, such as through impala-shell or using the create-demo-table -program included in the Kudu source tree. http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/README.adoc ---------------------------------------------------------------------- diff --git a/examples/java/insert-loadgen/README.adoc b/examples/java/insert-loadgen/README.adoc new file mode 100644 index 0000000..e2e697f --- /dev/null +++ b/examples/java/insert-loadgen/README.adoc @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + += Kudu Java insert loadgen README +:author: Kudu Team +:homepage: https://kudu.apache.org/ + +== Summary +Random insert load generator. This will insert as fast as it can into a +pre-existing table using AUTO_BACKGROUND_FLUSH mode. All fields are randomized. +The load generator will continue inserting until it is stopped or it encounters +an error. This load generator is single-threaded. + +To build and run, do the following: + +[source,bash] +---- +$ mvn verify +$ java -jar target/kudu-insert-loadgen-1.0-SNAPSHOT.jar master_addresses table_name +---- + +where `master_addresses` is a CSV of the master addresses for your Kudu cluster +and `table_name` is the name of a pre-existing table. For example, + +[source,bash] +---- +$ java -jar target/kudu-insert-loadgen-1.0-SNAPSHOT.jar localhost:7051 test_table +---- + +This program will not create the "test_table" table. You must do that +via other means, such as through impala-shell. http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/pom.xml ---------------------------------------------------------------------- diff --git a/examples/java/insert-loadgen/pom.xml b/examples/java/insert-loadgen/pom.xml index 09bc0d1..b988465 100644 --- a/examples/java/insert-loadgen/pom.xml +++ b/examples/java/insert-loadgen/pom.xml @@ -1,10 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +--> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> <modelVersion>4.0.0</modelVersion> - <groupId>kudu-examples</groupId> + <groupId>org.apache.kudu</groupId> <artifactId>kudu-insert-loadgen</artifactId> <packaging>jar</packaging> - <version>0.1-SNAPSHOT</version> + <version>1.0-SNAPSHOT</version> <name>Random Insert Load Generator for Kudu</name> <build> @@ -12,7 +32,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>2.3.1</version> + <version>3.6.2</version> <configuration> <source>1.7</source> <target>1.7</target> @@ -21,7 +41,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> - <version>2.4</version> + <version>3.0.0</version> <executions> <execution> <phase>package</phase> @@ -31,7 +51,7 @@ <configuration> <transformers> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>org.kududb.examples.loadgen.InsertLoadgen</mainClass> + <mainClass>org.apache.kudu.examples.InsertLoadgen</mainClass> </transformer> </transformers> </configuration> @@ -41,32 +61,19 @@ </plugins> </build> - <repositories> - <repository> - <id>cdh.repo</id> - <name>Cloudera Repositories</name> - <url>https://repository.cloudera.com/artifactory/cloudera-repos</url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> - <dependencies> - <dependency> <groupId>org.apache.kudu</groupId> <artifactId>kudu-client</artifactId> - <version>1.1.0</version> + <version>1.7.0</version> </dependency> - <!-- for logging messages --> + <!-- For logging messages. --> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-simple</artifactId> - <version>1.7.12</version> + <version>1.7.25</version> </dependency> - </dependencies> </project> http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java ---------------------------------------------------------------------- diff --git a/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java new file mode 100644 index 0000000..923d5bc --- /dev/null +++ b/examples/java/insert-loadgen/src/main/java/org/apache/kudu/examples/InsertLoadgen.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.kudu.examples; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.UUID; + +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.SessionConfiguration; + +public class InsertLoadgen { + private static class RandomDataGenerator { + private final Random rng; + private final int index; + private final Type type; + + /** + * Instantiate a random data generator for a specific field. + * @param index The numerical index of the column in the row schema + * @param type The type of the data at index {@code index} + */ + public RandomDataGenerator(int index, Type type) { + this.rng = new Random(); + this.index = index; + this.type = type; + } + + /** + * Add random data to the given row for the column at index {@code index} + * of type {@code type} + * @param row The row to add the field to + */ + void generateColumnData(PartialRow row) { + switch (type) { + case INT8: + row.addByte(index, (byte) rng.nextInt(Byte.MAX_VALUE)); + return; + case INT16: + row.addShort(index, (short)rng.nextInt(Short.MAX_VALUE)); + return; + case INT32: + row.addInt(index, rng.nextInt(Integer.MAX_VALUE)); + return; + case INT64: + case UNIXTIME_MICROS: + row.addLong(index, rng.nextLong()); + return; + case BINARY: + byte bytes[] = new byte[16]; + rng.nextBytes(bytes); + row.addBinary(index, bytes); + return; + case STRING: + row.addString(index, UUID.randomUUID().toString()); + return; + case BOOL: + row.addBoolean(index, rng.nextBoolean()); + return; + case FLOAT: + row.addFloat(index, rng.nextFloat()); + return; + case DOUBLE: + row.addDouble(index, rng.nextDouble()); + return; + case DECIMAL: + row.addDecimal(index, new BigDecimal(rng.nextDouble())); + return; + default: + throw new UnsupportedOperationException("Unknown type " + type); + } + } + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.err.println("Usage: InsertLoadgen master_addresses table_name"); + System.exit(1); + } + + String masterAddrs = args[0]; + String tableName = args[1]; + + try (KuduClient client = new KuduClient.KuduClientBuilder(masterAddrs).build()) { + KuduTable table = client.openTable(tableName); + Schema schema = table.getSchema(); + List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount()); + for (int i = 0; i < schema.getColumnCount(); i++) { + generators.add(new RandomDataGenerator(i, schema.getColumnByIndex(i).getType())); + } + + KuduSession session = client.newSession(); + session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); + for (int insertCount = 0; ; insertCount++) { + Insert insert = table.newInsert(); + PartialRow row = insert.getRow(); + for (int i = 0; i < schema.getColumnCount(); i++) { + generators.get(i).generateColumnData(row); + } + session.apply(insert); + + // Check for errors. This is done periodically since inserts are batched. + if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) { + throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString()); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/kudu/blob/cc8c5e3e/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java ---------------------------------------------------------------------- diff --git a/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java b/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java deleted file mode 100644 index 4b76c34..0000000 --- a/examples/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.kududb.examples.loadgen; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.UUID; - -import org.apache.kudu.Schema; -import org.apache.kudu.Type; -import org.apache.kudu.client.Insert; -import org.apache.kudu.client.KuduClient; -import org.apache.kudu.client.KuduSession; -import org.apache.kudu.client.KuduTable; -import org.apache.kudu.client.PartialRow; -import org.apache.kudu.client.SessionConfiguration; - -public class InsertLoadgen { - private static class RandomDataGenerator { - private final Random rng; - private final int index; - private final Type type; - - /** - * Instantiate a random data generator for a specific field. - * @param index The numerical index of the column in the row schema - * @param type The type of the data at index {@code index} - */ - public RandomDataGenerator(int index, Type type) { - this.rng = new Random(); - this.index = index; - this.type = type; - } - - /** - * Add random data to the given row for the column at index {@code index} - * of type {@code type} - * @param row The row to add the field to - */ - void generateColumnData(PartialRow row) { - switch (type) { - case INT8: - row.addByte(index, (byte) rng.nextInt(Byte.MAX_VALUE)); - return; - case INT16: - row.addShort(index, (short)rng.nextInt(Short.MAX_VALUE)); - return; - case INT32: - row.addInt(index, rng.nextInt(Integer.MAX_VALUE)); - return; - case INT64: - case UNIXTIME_MICROS: - row.addLong(index, rng.nextLong()); - return; - case BINARY: - byte bytes[] = new byte[16]; - rng.nextBytes(bytes); - row.addBinary(index, bytes); - return; - case STRING: - row.addString(index, UUID.randomUUID().toString()); - return; - case BOOL: - row.addBoolean(index, rng.nextBoolean()); - return; - case FLOAT: - row.addFloat(index, rng.nextFloat()); - return; - case DOUBLE: - row.addDouble(index, rng.nextDouble()); - return; - default: - throw new UnsupportedOperationException("Unknown type " + type); - } - } - } - - public static void main(String[] args) throws Exception { - if (args.length != 2) { - System.err.println("Usage: InsertLoadgen kudu_master_host kudu_table"); - System.exit(1); - } - - String masterHost = args[0]; - String tableName = args[1]; - - try (KuduClient client = new KuduClient.KuduClientBuilder(masterHost).build()) { - KuduTable table = client.openTable(tableName); - Schema schema = table.getSchema(); - List<RandomDataGenerator> generators = new ArrayList<>(schema.getColumnCount()); - for (int i = 0; i < schema.getColumnCount(); i++) { - generators.add(new RandomDataGenerator(i, schema.getColumnByIndex(i).getType())); - } - - KuduSession session = client.newSession(); - session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); - for (int insertCount = 0; ; insertCount++) { - Insert insert = table.newInsert(); - PartialRow row = insert.getRow(); - for (int i = 0; i < schema.getColumnCount(); i++) { - generators.get(i).generateColumnData(row); - } - session.apply(insert); - - if (insertCount % 1000 == 0 && session.countPendingErrors() > 0) { - throw new RuntimeException(session.getPendingErrors().getRowErrors()[0].toString()); - } - } - } - } -}
