RussellSpitzer commented on issue #15086:
URL: https://github.com/apache/iceberg/issues/15086#issuecomment-3774476364
I wrote another small test which seems to work, this goes in
TestVariantWriters.java
```java
@org.junit.jupiter.api.Test
public void testPartialShredding() throws IOException {
ByteBuffer partialMetadata =
VariantTestUtil.createMetadata(
ImmutableList.of("shred_a", "unshred_b", "shred_c",
"unshred_d"), true);
ByteBuffer partialObjectBuffer =
VariantTestUtil.createObject(
partialMetadata,
ImmutableMap.of(
"shred_a", Variants.of(42),
"unshred_b", Variants.of("unshredded-b"),
"shred_c", Variants.of(100),
"unshred_d", Variants.of("unshredded-d")));
VariantMetadata metadata = Variants.metadata(partialMetadata);
VariantObject variantObject = (VariantObject) Variants.value(metadata,
partialObjectBuffer);
Variant variant = Variant.of(metadata, variantObject);
Record record = RECORD.copy("id", 1, "var", variant);
VariantShreddingFunction partialShreddingFunc =
(id, name) -> {
if (name.equals("var")) {
GroupType fieldShredA =
org.apache.parquet.schema.Types.buildGroup(REQUIRED)
.optional(BINARY)
.named("value")
.addField(ParquetVariantUtil.toParquetSchema(variantObject.get("shred_a")))
.named("shred_a");
GroupType fieldShredC =
org.apache.parquet.schema.Types.buildGroup(REQUIRED)
.optional(BINARY)
.named("value")
.addField(ParquetVariantUtil.toParquetSchema(variantObject.get("shred_c")))
.named("shred_c");
return org.apache.parquet.schema.Types.buildGroup(OPTIONAL)
.addField(fieldShredA)
.addField(fieldShredC)
.named("typed_value");
}
return null;
};
Record actual = writeAndRead(partialShreddingFunc, record);
// Verify the record is equivalent
InternalTestHelpers.assertEquals(SCHEMA.asStruct(), record, actual);
// Extract the variant from the actual record and verify its structure
Variant actualVariant = (Variant) actual.getField("var");
assertThat(actualVariant).isNotNull();
VariantValue actualValue = actualVariant.value();
assertThat(actualValue).isInstanceOf(VariantObject.class);
VariantObject actualObject = (VariantObject) actualValue;
// Verify that shredded fields "shred_a" and "shred_c" are accessible
assertThat(actualObject.get("shred_a")).isNotNull();
assertThat(actualObject.get("shred_a").asPrimitive().get()).isEqualTo(42);
assertThat(actualObject.get("shred_c")).isNotNull();
assertThat(actualObject.get("shred_c").asPrimitive().get()).isEqualTo(100);
// Verify that unshredded fields "unshred_b" and "unshred_d" are
accessible
assertThat(actualObject.get("unshred_b")).isNotNull();
assertThat(actualObject.get("unshred_b").asPrimitive().get()).isEqualTo("unshredded-b");
assertThat(actualObject.get("unshred_d")).isNotNull();
assertThat(actualObject.get("unshred_d").asPrimitive().get()).isEqualTo("unshredded-d");
// Verify all 4 fields are present in the variant object
assertThat(actualObject.numFields()).isEqualTo(4);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
