This is an automated email from the ASF dual-hosted git repository.
achennaka pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 625ade7ae KUDU-1261 [Java] Array Datatype client support
625ade7ae is described below
commit 625ade7aea2adfaa7c8966b9846f960d2b61f3bf
Author: Abhishek Chennaka <[email protected]>
AuthorDate: Wed Sep 10 20:24:50 2025 -0700
KUDU-1261 [Java] Array Datatype client support
This patch adds DDL support to the Kudu Java client, including:
- Creating ColumnSchemas with Nested DataTypes
- Creating and altering tables with Nested DataType columns
Currently, only 1D array types are supported.
Corresponding tests have been added to:
- Validate the SchemaBuilder behavior
- Verify table schemas after creation and alteration
Excluded creating NESTED Type columns directly in tests that
create randomized Schema.
Change-Id: I02b5fc52d984d424b7be53e3cc4e3236a8d33aa9
Reviewed-on: http://gerrit.cloudera.org:8080/23419
Tested-by: Abhishek Chennaka <[email protected]>
Reviewed-by: Alexey Serbin <[email protected]>
---
.../org/apache/kudu/backup/TestKuduBackup.scala | 1 +
.../main/java/org/apache/kudu/ColumnSchema.java | 241 +++++++++++++++++++--
.../src/main/java/org/apache/kudu/Type.java | 6 +-
.../org/apache/kudu/client/ProtobufHelper.java | 45 +++-
.../java/org/apache/kudu/TestColumnSchema.java | 71 ++++++
.../org/apache/kudu/client/TestKuduClient.java | 69 ++++++
.../org/apache/kudu/client/TestPartialRow.java | 3 +-
.../spark/tools/DistributedDataGeneratorTest.scala | 5 +-
.../java/org/apache/kudu/test/ClientTestUtil.java | 37 ++++
9 files changed, 451 insertions(+), 27 deletions(-)
diff --git
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index cdf69811e..e2f2c481a 100644
---
a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++
b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -1031,6 +1031,7 @@ class TestKuduBackup extends KuduTestSuite {
val keyColumnCount = random.nextInt(columnCount) + 1 // At least one key.
val schemaGenerator = new SchemaGeneratorBuilder()
.random(random)
+ .excludeTypes(Type.NESTED)
.columnCount(columnCount)
.keyColumnCount(keyColumnCount)
.build()
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
index 26b679deb..3be19a028 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/ColumnSchema.java
@@ -52,6 +52,8 @@ public class ColumnSchema {
private final Common.DataType wireType;
private final String comment;
+ private final NestedTypeDescriptor nestedTypeDescriptor;
+
/**
* Specifies the encoding of data for a column on disk.
* Not all encodings are available for all data types.
@@ -111,7 +113,7 @@ public class ColumnSchema {
Object defaultValue, int desiredBlockSize, Encoding
encoding,
CompressionAlgorithm compressionAlgorithm,
ColumnTypeAttributes typeAttributes, Common.DataType
wireType,
- String comment) {
+ String comment, NestedTypeDescriptor
nestedTypeDescriptor) {
this.name = name;
this.type = type;
this.key = key;
@@ -127,6 +129,7 @@ public class ColumnSchema {
this.typeSize = type.getSize(typeAttributes);
this.wireType = wireType;
this.comment = comment;
+ this.nestedTypeDescriptor = nestedTypeDescriptor;
}
/**
@@ -248,6 +251,145 @@ public class ColumnSchema {
return comment;
}
+ /** Returns true if this column represents an array (1D). */
+ public boolean isArray() {
+ return nestedTypeDescriptor != null && nestedTypeDescriptor.isArray();
+ }
+
+ /** Return nested descriptor or null if none. */
+ public NestedTypeDescriptor getNestedTypeDescriptor() {
+ return nestedTypeDescriptor;
+ }
+
+
+ /**
+ * Top-level container for nested type descriptors (ARRAY, MAP, STRUCT, ...).
+ * Placed here as a static inner class to keep schema-related types together.
+ */
+ public static final class NestedTypeDescriptor {
+ private final Descriptor descriptor;
+
+ private NestedTypeDescriptor(Descriptor descriptor) {
+ this.descriptor = Objects.requireNonNull(descriptor, "descriptor");
+ }
+
+ public boolean isArray() {
+ return descriptor.array != null;
+ }
+
+ public ArrayTypeDescriptor getArrayDescriptor() {
+ if (!isArray()) {
+ throw new IllegalStateException("Not an array descriptor");
+ }
+ return descriptor.array;
+ }
+
+ private static final class Descriptor {
+ final ArrayTypeDescriptor array;
+
+ private Descriptor(ArrayTypeDescriptor array) {
+ if (array == null) {
+ throw new IllegalArgumentException("ArrayTypeDescriptor must not be
null");
+ }
+ this.array = array;
+ }
+
+ static Descriptor forArray(ArrayTypeDescriptor arr) {
+ return new Descriptor(arr);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof Descriptor)) {
+ return false;
+ }
+ Descriptor that = (Descriptor) o;
+ return Objects.equals(array, that.array);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(array);
+ }
+ }
+
+ public static NestedTypeDescriptor forArray(ArrayTypeDescriptor arr) {
+ return new NestedTypeDescriptor(Descriptor.forArray(arr));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof NestedTypeDescriptor)) {
+ return false;
+ }
+ NestedTypeDescriptor that = (NestedTypeDescriptor) o;
+ return Objects.equals(descriptor, that.descriptor);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(descriptor);
+ }
+
+ @Override
+ public String toString() {
+ if (isArray()) {
+ return descriptor.array.toString();
+ }
+ return "unknown-nested-type";
+ }
+ }
+
+ /**
+ * Descriptor for array-specific data.
+ */
+ public static final class ArrayTypeDescriptor {
+ private final Type elemType;
+
+ public ArrayTypeDescriptor(Type elemType) {
+ Objects.requireNonNull(elemType, "elemType");
+ if (elemType == Type.NESTED) {
+ // if element is nested, we would need nested descriptor in future
+ // for now disallow without explicit nested descriptor support
+ throw new IllegalArgumentException("Nested element without descriptor
not supported");
+ }
+ this.elemType = elemType;
+ }
+
+ public Type getElemType() {
+ return elemType;
+ }
+
+ @Override
+ public String toString() {
+ String base = elemType.getName();
+ return base + " 1D-ARRAY";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof ArrayTypeDescriptor)) {
+ return false;
+ }
+ ArrayTypeDescriptor that = (ArrayTypeDescriptor) o;
+ return elemType == that.elemType;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(elemType);
+ }
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -297,7 +439,7 @@ public class ColumnSchema {
private static final List<Type> TYPES_WITH_ATTRIBUTES =
Arrays.asList(Type.DECIMAL,
Type.VARCHAR);
private final String name;
- private final Type type;
+ private Type type;
private boolean key = false;
private boolean keyUnique = false;
private boolean nullable = false;
@@ -309,6 +451,11 @@ public class ColumnSchema {
private ColumnTypeAttributes typeAttributes = null;
private Common.DataType wireType = null;
private String comment = "";
+ // Nested descriptor captured when array(true) called
+ private NestedTypeDescriptor nestedTypeDescriptor = null;
+
+ private boolean isArray = false;
+
/**
* Constructor for the required parameters.
@@ -323,6 +470,10 @@ public class ColumnSchema {
Schema.getAutoIncrementingColumnName() + " is reserved by Kudu
engine");
}
this.name = name;
+ if (type == Type.NESTED) {
+ throw new IllegalArgumentException("Column " +
+ name + " cannot be set to NESTED type. Use
ColumnSchemaBuilder.array(true) instead");
+ }
this.type = type;
}
@@ -344,6 +495,7 @@ public class ColumnSchema {
this.typeAttributes = that.typeAttributes;
this.wireType = that.wireType;
this.comment = that.comment;
+ this.nestedTypeDescriptor = that.nestedTypeDescriptor;
}
/**
@@ -482,29 +634,82 @@ public class ColumnSchema {
return this;
}
+ /**
+ * Marks this builder as creating a 1D array column. This records the
intent,
+ * but does NOT immediately mutate the builder's declared 'type' into
NESTED.
+ * Final promotion/validation happens inside build().
+ */
+ public ColumnSchemaBuilder array(boolean isArray) {
+ if (isArray) {
+ if (this.type == Type.NESTED) {
+ throw new IllegalArgumentException("Builder.type must be scalar when
calling " +
+ "array(true)");
+ }
+ this.isArray = true;
+ } else {
+ this.isArray = false;
+ }
+ return this;
+ }
+
/**
* Builds a {@link ColumnSchema} using the passed parameters.
- * @return a new {@link ColumnSchema}
+ * If array(true) was called, this method will create ArrayTypeDescriptor
and
+ * NestedTypeDescriptor, promote the column to Type.NESTED and perform all
+ * necessary validation.
*/
public ColumnSchema build() {
- // Set the wire type if it wasn't explicitly set.
- if (wireType == null) {
- this.wireType = type.getDataType(typeAttributes);
+ final Type finalType;
+
+ if (this.isArray) {
+ // The builder's `type` currently holds the element (scalar) type.
+ if (this.type == Type.NESTED) {
+ throw new IllegalArgumentException("Builder.type must be scalar when
creating " +
+ "an array column");
+ }
+ ColumnSchema.ArrayTypeDescriptor arrDesc = new
ColumnSchema.ArrayTypeDescriptor(this.type);
+ nestedTypeDescriptor =
ColumnSchema.NestedTypeDescriptor.forArray(arrDesc);
+ finalType = Type.NESTED;
+ } else {
+ nestedTypeDescriptor = null;
+ finalType = this.type;
}
- if (type == Type.VARCHAR) {
- if (typeAttributes == null || !typeAttributes.hasLength() ||
- typeAttributes.getLength() < CharUtil.MIN_VARCHAR_LENGTH ||
- typeAttributes.getLength() > CharUtil.MAX_VARCHAR_LENGTH) {
- throw new IllegalArgumentException(
- String.format("VARCHAR's length must be set and between %d and %d",
- CharUtil.MIN_VARCHAR_LENGTH,
CharUtil.MAX_VARCHAR_LENGTH));
+
+ // Validate type attributes:
+ // - For scalar VARCHAR: typeAttributes.length must be set and in bounds.
+ // - For array element VARCHAR: same requirements apply to column-level
typeAttributes
+ if (finalType != Type.NESTED) {
+ if (finalType == Type.VARCHAR) {
+ if (typeAttributes == null || !typeAttributes.hasLength() ||
+ typeAttributes.getLength() < CharUtil.MIN_VARCHAR_LENGTH ||
+ typeAttributes.getLength() > CharUtil.MAX_VARCHAR_LENGTH) {
+ throw new IllegalArgumentException(
+ String.format("VARCHAR's length must be set and between %d and
%d",
+ CharUtil.MIN_VARCHAR_LENGTH, CharUtil.MAX_VARCHAR_LENGTH));
+ }
+ }
+ } else {
+ // For arrays, element type rules apply.
+ Type elemType =
nestedTypeDescriptor.getArrayDescriptor().getElemType();
+ if (elemType == Type.VARCHAR) {
+ if (typeAttributes == null || !typeAttributes.hasLength() ||
+ typeAttributes.getLength() < CharUtil.MIN_VARCHAR_LENGTH ||
+ typeAttributes.getLength() > CharUtil.MAX_VARCHAR_LENGTH) {
+ throw new IllegalArgumentException(
+ String.format("Array element VARCHAR's length must be set and
between %d and %d",
+ CharUtil.MIN_VARCHAR_LENGTH, CharUtil.MAX_VARCHAR_LENGTH));
+ }
}
}
- return new ColumnSchema(name, type, key, keyUnique, nullable, immutable,
- /* autoIncrementing */false, defaultValue,
- desiredBlockSize, encoding, compressionAlgorithm,
- typeAttributes, wireType, comment);
+ if (wireType == null) {
+ this.wireType = finalType.getDataType(typeAttributes);
+ }
+
+ // Finally build immutable ColumnSchema with nested descriptor if any.
+ return new ColumnSchema(name, finalType, key, keyUnique, nullable,
immutable,
+ /* autoIncrementing */ false, defaultValue, desiredBlockSize,
encoding,
+ compressionAlgorithm, typeAttributes, wireType, comment,
nestedTypeDescriptor);
}
}
@@ -591,7 +796,7 @@ public class ColumnSchema {
/* nullable */false, /* immutable */false,
/* autoIncrementing */true, /* defaultValue
*/null,
desiredBlockSize, encoding, compressionAlgorithm,
- /* typeAttributes */null, wireType, comment);
+ /* typeAttributes */null, wireType, comment,
null);
}
}
}
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/Type.java
b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
index 9d550b66c..8c91685d0 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/Type.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/Type.java
@@ -50,7 +50,8 @@ public enum Type {
UNIXTIME_MICROS(DataType.UNIXTIME_MICROS, "unixtime_micros"),
DECIMAL(Arrays.asList(DataType.DECIMAL32, DataType.DECIMAL64,
DataType.DECIMAL128), "decimal"),
VARCHAR(DataType.VARCHAR, "varchar"),
- DATE(DataType.DATE, "date");
+ DATE(DataType.DATE, "date"),
+ NESTED(DataType.NESTED, "nested");
private final ImmutableList<DataType> dataTypes;
private final String name;
@@ -148,6 +149,7 @@ public enum Type {
case STRING:
case BINARY:
case VARCHAR:
+ case NESTED:
return 8 + 8; // offset then string length
case BOOL:
case INT8:
@@ -204,6 +206,8 @@ public enum Type {
case DECIMAL64:
case DECIMAL128:
return DECIMAL;
+ case NESTED:
+ return NESTED;
default:
throw new IllegalArgumentException("the provided data type doesn't map
" +
"to any known one: " + type.getDescriptorForType().getFullName());
diff --git
a/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
index a9c015bb0..7e605adec 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/ProtobufHelper.java
@@ -17,6 +17,7 @@
package org.apache.kudu.client;
+import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;
import java.math.BigDecimal;
@@ -132,6 +133,28 @@ public class ProtobufHelper {
!column.getComment().isEmpty()) {
schemaBuilder.setComment(column.getComment());
}
+
+ // Handle nested/array column: populate NestedDataTypePB.array.type with
the type of
+ // array elements
+ if (column.isArray()) {
+ ColumnSchema.NestedTypeDescriptor nested =
column.getNestedTypeDescriptor();
+ checkState(nested == null || nested.isArray(),
+ "Column reports isArray() but nested descriptor is not ARRAY");
+
+ ColumnSchema.ArrayTypeDescriptor arr = nested.getArrayDescriptor();
+ Type elemType = arr.getElemType();
+
+ Common.NestedDataTypePB.ArrayTypeDescriptor.Builder arrPb =
+ Common.NestedDataTypePB.ArrayTypeDescriptor.newBuilder();
+
+ Common.DataType elemPbType =
elemType.getDataType(column.getTypeAttributes());
+ arrPb.setType(elemPbType);
+
+ Common.NestedDataTypePB.Builder nestedPb =
Common.NestedDataTypePB.newBuilder();
+ nestedPb.setArray(arrPb);
+
+ schemaBuilder.setNestedType(nestedPb);
+ }
return schemaBuilder.build();
}
@@ -171,13 +194,25 @@ public class ProtobufHelper {
.build();
}
- Type type = Type.getTypeForDataType(pb.getType());
ColumnTypeAttributes typeAttributes = pb.hasTypeAttributes() ?
pbToColumnTypeAttributes(pb.getTypeAttributes()) : null;
- Object defaultValue = pb.hasWriteDefaultValue() ?
- byteStringToObject(type, typeAttributes, pb.getWriteDefaultValue()) :
null;
- ColumnSchema.ColumnSchemaBuilder csb =
- new ColumnSchema.ColumnSchemaBuilder(pb.getName(), type);
+ Object defaultValue;
+ ColumnSchema.ColumnSchemaBuilder csb;
+ // Set the name, type and populate default value for nested column.
+ if (pb.hasNestedType() && pb.getNestedType().hasArray()) {
+ Common.NestedDataTypePB.ArrayTypeDescriptor arrPb =
pb.getNestedType().getArray();
+ Type elemType = Type.getTypeForDataType(arrPb.getType());
+ csb = new ColumnSchema.ColumnSchemaBuilder(pb.getName(), elemType);
+ csb.array(true);
+ // TODO(achennaka): Re-visit the read and write default implementation
for nested types.
+ defaultValue = pb.hasWriteDefaultValue() ?
+ byteStringToObject(elemType, typeAttributes,
pb.getWriteDefaultValue()) : null;
+ } else {
+ Type type = Type.getTypeForDataType(pb.getType());
+ csb = new ColumnSchema.ColumnSchemaBuilder(pb.getName(), type);
+ defaultValue = pb.hasWriteDefaultValue() ?
+ byteStringToObject(type, typeAttributes,
pb.getWriteDefaultValue()) : null;
+ }
if (pb.getIsKey() && isKeyUnique) {
csb.key(true);
} else {
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
b/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
index c1fd3c9e5..c11300ecf 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/TestColumnSchema.java
@@ -159,4 +159,75 @@ public class TestColumnSchema {
Assert.assertTrue(thrown.getMessage().contains("Column name " +
Schema.getAutoIncrementingColumnName() + " is reserved by Kudu
engine"));
}
+
+ @Test
+ public void testArrayTypeColumn() {
+ // STRING[]
+ ColumnSchema strArrayCol = new ColumnSchema.ColumnSchemaBuilder("str_arr",
Type.STRING)
+ .array(true) // promotes to NESTED with element type STRING
+ .nullable(true) // the array itself can be null
+ .build();
+
+ assertEquals("str_arr", strArrayCol.getName());
+ assertEquals(Type.NESTED, strArrayCol.getType());
+ Assert.assertTrue(strArrayCol.isNullable());
+ assertEquals(Type.STRING,
+
strArrayCol.getNestedTypeDescriptor().getArrayDescriptor().getElemType());
+
+ // DECIMAL(10,4)[]
+ ColumnTypeAttributes decimalAttrs = new
ColumnTypeAttributes.ColumnTypeAttributesBuilder()
+ .precision(10)
+ .scale(4)
+ .build();
+
+ ColumnSchema decimalArrayCol = new
ColumnSchema.ColumnSchemaBuilder("dec_arr", Type.DECIMAL)
+ .typeAttributes(decimalAttrs)
+ .array(true)
+ .nullable(true)
+ .build();
+
+ assertEquals("dec_arr", decimalArrayCol.getName());
+ assertEquals(Type.NESTED, decimalArrayCol.getType());
+ Assert.assertTrue(decimalArrayCol.isNullable());
+ assertEquals(Type.DECIMAL,
+
decimalArrayCol.getNestedTypeDescriptor().getArrayDescriptor().getElemType());
+
+ // INT32[] (non-nullable)
+ ColumnSchema intArrayCol = new ColumnSchema.ColumnSchemaBuilder("int_arr",
Type.INT32)
+ .array(true)
+ .nullable(false)
+ .build();
+
+ assertEquals("int_arr", intArrayCol.getName());
+ assertEquals(Type.NESTED, intArrayCol.getType());
+ Assert.assertFalse(intArrayCol.isNullable());
+ assertEquals(Type.INT32,
+
intArrayCol.getNestedTypeDescriptor().getArrayDescriptor().getElemType());
+
+ // VARCHAR(50)[]
+ ColumnTypeAttributes varcharAttrs = new
ColumnTypeAttributes.ColumnTypeAttributesBuilder()
+ .length(50)
+ .build();
+
+ ColumnSchema varcharArrayCol = new
ColumnSchema.ColumnSchemaBuilder("varchar_arr", Type.VARCHAR)
+ .typeAttributes(varcharAttrs)
+ .array(true)
+ .nullable(true)
+ .build();
+
+ assertEquals("varchar_arr", varcharArrayCol.getName());
+ assertEquals(Type.NESTED, varcharArrayCol.getType());
+ Assert.assertTrue(varcharArrayCol.isNullable());
+ assertEquals(Type.VARCHAR,
+
varcharArrayCol.getNestedTypeDescriptor().getArrayDescriptor().getElemType());
+
+ // Test constructor restriction: cannot pass NESTED directly
+ IllegalArgumentException thrown =
Assert.assertThrows(IllegalArgumentException.class, () ->
+ new ColumnSchema.ColumnSchemaBuilder("nested", Type.NESTED)
+ );
+ Assert.assertTrue(thrown.getMessage().contains(
+ "Column nested cannot be set to NESTED type. Use
ColumnSchemaBuilder.array(true) instead"
+ ));
+
+ }
}
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index 8f7073b13..be42de4ec 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -26,6 +26,7 @@ import static
org.apache.kudu.test.ClientTestUtil.countRowsInScan;
import static org.apache.kudu.test.ClientTestUtil.createBasicSchemaInsert;
import static org.apache.kudu.test.ClientTestUtil.createManyStringsSchema;
import static org.apache.kudu.test.ClientTestUtil.createManyVarcharsSchema;
+import static org.apache.kudu.test.ClientTestUtil.createSchemaWithArrayColumns;
import static
org.apache.kudu.test.ClientTestUtil.createSchemaWithBinaryColumns;
import static org.apache.kudu.test.ClientTestUtil.createSchemaWithDateColumns;
import static
org.apache.kudu.test.ClientTestUtil.createSchemaWithDecimalColumns;
@@ -1173,6 +1174,74 @@ public class TestKuduClient {
client.deleteTable(TABLE_NAME);
}
+ /**
+ * Test creating a table schema with an array type column.
+ */
+ @Test(timeout = 100000)
+ public void testArrayTypeColumnCreationAndAlter() throws Exception {
+ Schema schema = createSchemaWithArrayColumns();
+ client.createTable(TABLE_NAME, schema, getBasicCreateTableOptions());
+
+ KuduTable table = client.openTable(TABLE_NAME);
+ Schema returnedSchema = table.getSchema();
+
+ // Verify initial columns (as in your original test)
+ assertEquals(5, returnedSchema.getColumnCount());
+ assertTrue(returnedSchema.getColumn("int_arr").isArray());
+ assertFalse(returnedSchema.getColumn("int_arr").isKey());
+ assertTrue(returnedSchema.getColumn("nullable_int_arr").isArray());
+ assertTrue(returnedSchema.getColumn("dec_arr").isArray());
+ assertTrue(returnedSchema.getColumn("str_arr").isArray());
+
+ // ---- Sub-scenario: Alter table ----
+ AlterTableOptions alter = new AlterTableOptions();
+
+ // Add new array columns
+ ColumnSchema newFloatArray =
+ new ColumnSchema.ColumnSchemaBuilder("string_arr", Type.STRING)
+ .array(true)
+ .nullable(true)
+ .build();
+ alter.addColumn(newFloatArray);
+
+ ColumnSchema newIntCol =
+ new ColumnSchema.ColumnSchemaBuilder("int_col", Type.INT32)
+ .array(false)
+ .nullable(true)
+ .build();
+ alter.addColumn(newIntCol);
+
+ // Drop existing array column
+ alter.dropColumn("nullable_int_arr");
+
+ client.alterTable(TABLE_NAME, alter);
+
+ // Validate altered schema
+ table = client.openTable(TABLE_NAME);
+ returnedSchema = table.getSchema();
+
+ // Column count should stay 6 (added two, dropped one)
+ assertEquals(6, returnedSchema.getColumnCount());
+
+ // Check the new columns
+ ColumnSchema floatArr = returnedSchema.getColumn("string_arr");
+ assertTrue(floatArr.isArray());
+ assertTrue(floatArr.isNullable());
+ assertEquals(Type.NESTED, floatArr.getType());
+ assertEquals(Type.STRING,
+
floatArr.getNestedTypeDescriptor().getArrayDescriptor().getElemType());
+ ColumnSchema intCol = returnedSchema.getColumn("int_col");
+ assertFalse(intCol.isArray());
+ assertTrue(intCol.isNullable());
+ assertNotEquals(Type.NESTED, intCol.getType());
+ assertEquals(Type.INT32, intCol.getType());
+
+ // Check dropped column is gone
+ assertFalse(returnedSchema.hasColumn("nullable_int_arr"));
+
+ client.deleteTable(TABLE_NAME);
+ }
+
/**
* Test inserting and retrieving rows from a table that has a range partition
* with custom hash schema.
diff --git
a/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartialRow.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartialRow.java
index e2e01978e..df8dd4e70 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartialRow.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestPartialRow.java
@@ -489,7 +489,8 @@ public class TestPartialRow {
// Shift the type one position to force the wrong type for all types.
private Type getShiftedType(Type type) {
- int shiftedPosition = (type.ordinal() + 1) % Type.values().length;
+ // TODO(achennaka) : remove the stopgap once serdes of array datatype is
implemented.
+ int shiftedPosition = (type.ordinal() + 1) % (Type.values().length - 1);
return Type.values()[shiftedPosition];
}
diff --git
a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/DistributedDataGeneratorTest.scala
b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/DistributedDataGeneratorTest.scala
index 4c7bd4681..e7486a39e 100644
---
a/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/DistributedDataGeneratorTest.scala
+++
b/java/kudu-spark-tools/src/test/scala/org/apache/kudu/spark/tools/DistributedDataGeneratorTest.scala
@@ -36,8 +36,9 @@ class DistributedDataGeneratorTest extends KuduTestSuite {
private val generator = new SchemaGenerator.SchemaGeneratorBuilder()
.random(RandomUtils.getRandom)
- // These types don't have enough values to prevent collisions.
- .excludeTypes(Type.BOOL, Type.INT8)
+ // BOOL and INT8 types don't have enough values to prevent collisions.
+ // NESTED cannot be set directly as a Type.
+ .excludeTypes(Type.BOOL, Type.INT8, Type.NESTED)
// Ensure decimals have enough values to prevent collisions.
.precisionRange(DecimalUtil.MAX_DECIMAL32_PRECISION,
DecimalUtil.MAX_DECIMAL_PRECISION)
.build()
diff --git
a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/ClientTestUtil.java
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/ClientTestUtil.java
index 3cd85d015..888a12b8b 100644
---
a/java/kudu-test-utils/src/main/java/org/apache/kudu/test/ClientTestUtil.java
+++
b/java/kudu-test-utils/src/main/java/org/apache/kudu/test/ClientTestUtil.java
@@ -513,4 +513,41 @@ public abstract class ClientTestUtil {
.build());
return new Schema(columns);
}
+
+ public static Schema createSchemaWithArrayColumns() {
+ ArrayList<ColumnSchema> columns = new ArrayList<>();
+ // Primary key column
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32)
+ .key(true)
+ .build());
+
+ // Non-nullable 1D array of INT32
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("int_arr", Type.INT32)
+ .array(true)
+ .nullable(false)
+ .build());
+
+ // Nullable 1D array of INT32
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("nullable_int_arr",
Type.INT32)
+ .array(true)
+ .nullable(true)
+ .build());
+
+ // Nullable 1D array of DECIMAL with precision/scale
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("dec_arr", Type.DECIMAL)
+ .array(true)
+ .nullable(true)
+ .typeAttributes(new
ColumnTypeAttributes.ColumnTypeAttributesBuilder()
+ .precision(10)
+ .scale(2)
+ .build())
+ .build());
+
+ // Nullable 1D array of STRING
+ columns.add(new ColumnSchema.ColumnSchemaBuilder("str_arr", Type.STRING)
+ .array(true)
+ .nullable(true)
+ .build());
+ return new Schema(columns);
+ }
}
