Update to 0.10.0 API, update Spark examples This changes the poms and Java code to use the new org.apache.kudu package name, updates to 0.10.0, and updates the spark example in collectl/README to use Spark SQL instead of RDDs.
Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/3349b9a6 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/3349b9a6 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/3349b9a6 Branch: refs/heads/master Commit: 3349b9a690da22524551da6f4aa410941acbfa52 Parents: 02d6eaa Author: Todd Lipcon <[email protected]> Authored: Tue Aug 23 12:10:00 2016 -0700 Committer: Todd Lipcon <[email protected]> Committed: Tue Aug 23 12:10:00 2016 -0700 ---------------------------------------------------------------------- java/collectl/README | 32 ++++++++------------ java/collectl/pom.xml | 4 +-- .../examples/collectl/KuduCollectlExample.java | 24 +++++++-------- java/insert-loadgen/pom.xml | 4 +-- .../kududb/examples/loadgen/InsertLoadgen.java | 16 +++++----- java/java-sample/pom.xml | 4 +-- .../java/org/kududb/examples/sample/Sample.java | 8 ++--- 7 files changed, 43 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/collectl/README ---------------------------------------------------------------------- diff --git a/java/collectl/README b/java/collectl/README index e8b7c40..74a2fd7 100644 --- a/java/collectl/README +++ b/java/collectl/README @@ -60,29 +60,23 @@ You can install it by running: Download the Kudu MR jar and run Spark with it on the classpath: - wget http://d2106.halxg.cloudera.com/kudu-2015-08-13-1/org/kududb/kudu-mapreduce/0.1.0-SNAPSHOT/kudu-mapreduce-0.1.0-20150813.213926-1-jar-with-dependencies.jar - spark-shell --jars kudu-mapred*jar + wget https://repository.cloudera.com/artifactory/cloudera-repos/org/apache/kudu/kudu-spark_2.10/0.10.0/kudu-spark_2.10-0.10.0.jar + spark-shell --jars kudu-spark*jar You can then paste this example script: - import org.kududb.mapreduce._ - import org.apache.hadoop.conf.Configuration - import org.kududb.client._ - import org.apache.hadoop.io.NullWritable; - - val conf = new Configuration - conf.set("kudu.mapreduce.master.address", "quickstart.cloudera"); - conf.set("kudu.mapreduce.input.table", "metrics"); - conf.set("kudu.mapreduce.column.projection", "host,metric,timestamp,value"); - val kuduRdd = sc.newAPIHadoopRDD(conf, classOf[KuduTableInputFormat], classOf[NullWritable], classOf[RowResult]) - + import org.apache.kudu.spark.kudu._ + + val df = sqlContext.read.options(Map( + "kudu.master" -> "quickstart.cloudera", + "kudu.table" -> "metrics")).kudu + df.registerTempTable("metrics") + // Print the first five values - kuduRdd.values.map(r => r.rowToString()).take(5).foreach(x => print(x + "\n")) + sqlContext.sql("select * from metrics limit 5").show() // Calculate the average value of every host/metric pair - (kuduRdd.values.map(r => (r.getString(0) + "/" + r.getString(1), r.getDouble(3))). - mapValues(x => (x, 1)) - .reduceByKey((x, y) => (x._1 + y._1, x._2 + y._2)) - .mapValues(result => result._1) - .take(5)) + sqlContext.sql("select host, metric, avg(value) from metrics group by host, metric").show() +Note that if you are still running the 'collectl' command above, you can see +the data changing in real time by re-running the queries. http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/collectl/pom.xml ---------------------------------------------------------------------- diff --git a/java/collectl/pom.xml b/java/collectl/pom.xml index 65d1f36..0744dec 100644 --- a/java/collectl/pom.xml +++ b/java/collectl/pom.xml @@ -55,9 +55,9 @@ <dependencies> <dependency> - <groupId>org.kududb</groupId> + <groupId>org.apache.kudu</groupId> <artifactId>kudu-client</artifactId> - <version>0.9.0</version> + <version>0.10.0</version> </dependency> <!-- for logging messages --> http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/collectl/src/main/java/org/kududb/examples/collectl/KuduCollectlExample.java ---------------------------------------------------------------------- diff --git a/java/collectl/src/main/java/org/kududb/examples/collectl/KuduCollectlExample.java b/java/collectl/src/main/java/org/kududb/examples/collectl/KuduCollectlExample.java index 3e1ae29..f7c7d1c 100644 --- a/java/collectl/src/main/java/org/kududb/examples/collectl/KuduCollectlExample.java +++ b/java/collectl/src/main/java/org/kududb/examples/collectl/KuduCollectlExample.java @@ -10,18 +10,18 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import org.kududb.ColumnSchema; -import org.kududb.ColumnSchema.ColumnSchemaBuilder; -import org.kududb.Schema; -import org.kududb.Type; -import org.kududb.client.CreateTableOptions; -import org.kududb.client.Insert; -import org.kududb.client.KuduClient; -import org.kududb.client.KuduSession; -import org.kududb.client.KuduTable; -import org.kududb.client.OperationResponse; -import org.kududb.client.RowError; -import org.kududb.client.SessionConfiguration.FlushMode; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.CreateTableOptions; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.OperationResponse; +import org.apache.kudu.client.RowError; +import org.apache.kudu.client.SessionConfiguration.FlushMode; public class KuduCollectlExample { http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/insert-loadgen/pom.xml ---------------------------------------------------------------------- diff --git a/java/insert-loadgen/pom.xml b/java/insert-loadgen/pom.xml index fddcdf9..e7a4872 100644 --- a/java/insert-loadgen/pom.xml +++ b/java/insert-loadgen/pom.xml @@ -55,9 +55,9 @@ <dependencies> <dependency> - <groupId>org.kududb</groupId> + <groupId>org.apache.kudu</groupId> <artifactId>kudu-client</artifactId> - <version>0.9.0</version> + <version>0.10.0</version> </dependency> <!-- for logging messages --> http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java ---------------------------------------------------------------------- diff --git a/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java index f7438c0..2b32af1 100644 --- a/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java +++ b/java/insert-loadgen/src/main/java/org/kududb/examples/loadgen/InsertLoadgen.java @@ -1,13 +1,13 @@ package org.kududb.examples.loadgen; -import org.kududb.Schema; -import org.kududb.Type; -import org.kududb.client.Insert; -import org.kududb.client.KuduClient; -import org.kududb.client.KuduSession; -import org.kududb.client.KuduTable; -import org.kududb.client.PartialRow; -import org.kududb.client.SessionConfiguration; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.Insert; +import org.apache.kudu.client.KuduClient; +import org.apache.kudu.client.KuduSession; +import org.apache.kudu.client.KuduTable; +import org.apache.kudu.client.PartialRow; +import org.apache.kudu.client.SessionConfiguration; import java.util.ArrayList; import java.util.List; http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/java-sample/pom.xml ---------------------------------------------------------------------- diff --git a/java/java-sample/pom.xml b/java/java-sample/pom.xml index 2f66c99..a948bf3 100644 --- a/java/java-sample/pom.xml +++ b/java/java-sample/pom.xml @@ -55,9 +55,9 @@ <dependencies> <dependency> - <groupId>org.kududb</groupId> + <groupId>org.apache.kudu</groupId> <artifactId>kudu-client</artifactId> - <version>0.9.0</version> + <version>0.10.0</version> </dependency> <!-- for logging messages --> http://git-wip-us.apache.org/repos/asf/kudu/blob/3349b9a6/java/java-sample/src/main/java/org/kududb/examples/sample/Sample.java ---------------------------------------------------------------------- diff --git a/java/java-sample/src/main/java/org/kududb/examples/sample/Sample.java b/java/java-sample/src/main/java/org/kududb/examples/sample/Sample.java index bd4b37b..5c5285f 100644 --- a/java/java-sample/src/main/java/org/kududb/examples/sample/Sample.java +++ b/java/java-sample/src/main/java/org/kududb/examples/sample/Sample.java @@ -1,9 +1,9 @@ package org.kududb.examples.sample; -import org.kududb.ColumnSchema; -import org.kududb.Schema; -import org.kududb.Type; -import org.kududb.client.*; +import org.apache.kudu.ColumnSchema; +import org.apache.kudu.Schema; +import org.apache.kudu.Type; +import org.apache.kudu.client.*; import java.util.ArrayList; import java.util.List;
