[
https://issues.apache.org/jira/browse/KUDU-1820?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15994285#comment-15994285
]
Cam Quoc Mach commented on KUDU-1820:
-------------------------------------
Hi Ryan, there is a simply way to create a table with range partitioning. Here
is how I do:
val kuduMaster = "localhost";
val kuduClient = new KuduClient.KuduClientBuilder(kuduMaster).build();
val tableName = "TableRangePartition";
var columns = new MutableList[ColumnSchema]();
columns.+=(new ColumnSchema.ColumnSchemaBuilder("id",
Type.INT32).nullable(false).key(true).build());
columns.+=(new ColumnSchema.ColumnSchemaBuilder("value",
Type.STRING).nullable(false).build());
val colSchema = new Schema(columns);
val tableOptions = new
CreateTableOptions().setRangePartitionColumns(List("id"))
.addHashPartitions(List("id"), 3)
.setNumReplicas(1);
var bounds = new MutableList[Pair[Integer, Integer]];
bounds.+=(new Pair[Integer, Integer](1, 10));
bounds.+=(new Pair[Integer, Integer](11, 20));
bounds.+=(new Pair[Integer, Integer](21, 30));
for (bound <- bounds) {
val lower = colSchema.newPartialRow();
val upper = colSchema.newPartialRow();
lower.addInt("id", bound.getFirst());
upper.addInt("id", bound.getSecond());
tableOptions.addRangePartition(lower, upper);
}
val kuduTable = kuduClient.createTable(tableName, colSchema,
tableOptions);
Let me know if it is something you look for, or something else?
> Improve KuduContext with Range Partitioning
> -------------------------------------------
>
> Key: KUDU-1820
> URL: https://issues.apache.org/jira/browse/KUDU-1820
> Project: Kudu
> Issue Type: Improvement
> Components: spark
> Affects Versions: 1.0.1
> Reporter: Ryan Bosshart
> Assignee: Cam Quoc Mach
> Priority: Minor
>
> With hash partitions, I use the KuduContext like so:
> kuduContext.createTable(modifiedTable, predictions.schema,
> Seq("movieid","userid"),
> new CreateTableOptions().addHashPartitions(ImmutableList.of("movieid"),
> 3).setNumReplicas(1))
> There isn't a clean way to use KuduContext with range partitions however. I
> have it working below, but CreateTableOptions and KuduContext both take a
> schema, but of different formats (one a list of ColumnSchema, the other a
> StructType).
> val fixSchema: Schema = {
> val columns = ImmutableList.of(
> new ColumnSchemaBuilder("clordid", Type.STRING).key(true).build(),
> new ColumnSchemaBuilder("transacttime", Type.INT64).key(true).build(),
> ....
> new ColumnSchemaBuilder("lastupdated", Type.INT64).key(false).build())
> new Schema(columns)
> }
> val schema =
> StructType(
> StructField("clordid", StringType, false) ::
> StructField("transacttime", LongType, false) ::
> ....
> StructField("lastupdated", LongType, true) :: Nil)
> val kuduContext = new KuduContext(kuduMaster)
> val options = new CreateTableOptions()
> .setRangePartitionColumns(ImmutableList.of("transacttime"))
> .addHashPartitions(ImmutableList.of("clordid"), 3)
> .setNumReplicas(1)
> val today = new DateTime().withTimeAtStartOfDay()
> val dayInMillis = TimeUnit.MILLISECONDS.convert(1, TimeUnit.DAYS)
> for (i <- 0 until numberOfDays ){
> val lowerBound = fixSchema.newPartialRow()
> val lbMillis = today.plusDays(i).getMillis
> lowerBound.addLong("transacttime", lbMillis)
> val upperBound = fixSchema.newPartialRow()
> upperBound.addLong("transacttime", (lbMillis+dayInMillis-1))
> options.addRangePartition(lowerBound, upperBound)
> }
> kuduContext.createTable(tableName, schema,
> Seq("clordid","transacttime"),options)
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
