nssalian commented on code in PR #17342:
URL: https://github.com/apache/iceberg/pull/17342#discussion_r3641940148
##########
parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java:
##########
@@ -273,6 +274,178 @@ public void testShreddedBinaryUpperBoundOverflow() throws
IOException {
.isNull();
}
+ @Test
+ public void testShreddedBinaryBoundsTruncateLength() throws IOException {
+ // a per-column truncate(8) overrides the default 16-byte truncation
+ byte[] bytes = new byte[20];
+ for (int i = 0; i < bytes.length; i += 1) {
+ bytes[i] = (byte) (i + 1);
+ }
+ VariantValue value = Variants.of(ByteBuffer.wrap(bytes));
+
+ MetricsConfig metricsConfig =
+ MetricsConfig.from(
+ ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "truncate(8)"),
+ SCHEMA,
+ null);
+
+ Metrics metrics =
+ writeParquetWithMetricsConfig(
+ (id, name) -> ParquetVariantUtil.toParquetSchema(value),
+ metricsConfig,
+ Variant.of(EMPTY, value),
+ Variant.of(EMPTY, Variants.ofNull()),
+ null);
+
+ assertThat(metrics.lowerBounds().get(2))
+ .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+
.isEqualTo(Variants.of(BinaryUtil.truncateBinaryMin(ByteBuffer.wrap(bytes),
8)));
+
+ assertThat(metrics.upperBounds().get(2))
+ .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+
.isEqualTo(Variants.of(BinaryUtil.truncateBinaryMax(ByteBuffer.wrap(bytes),
8)));
+ }
+
+ @Test
+ public void testShreddedBinaryBoundsFull() throws IOException {
+ // full mode leaves the bounds untruncated
+ byte[] bytes = new byte[20];
+ for (int i = 0; i < bytes.length; i += 1) {
+ bytes[i] = (byte) (i + 1);
+ }
+ VariantValue value = Variants.of(ByteBuffer.wrap(bytes));
+
+ MetricsConfig metricsConfig =
+ MetricsConfig.from(
+ ImmutableMap.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX +
"var", "full"),
+ SCHEMA,
+ null);
+
+ Metrics metrics =
+ writeParquetWithMetricsConfig(
+ (id, name) -> ParquetVariantUtil.toParquetSchema(value),
+ metricsConfig,
+ Variant.of(EMPTY, value),
+ Variant.of(EMPTY, Variants.ofNull()),
+ null);
+
+ assertThat(metrics.lowerBounds().get(2))
+ .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+ .isEqualTo(Variants.of(ByteBuffer.wrap(bytes)));
+
+ assertThat(metrics.upperBounds().get(2))
+ .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD))
+ .isEqualTo(Variants.of(ByteBuffer.wrap(bytes)));
+ }
+
+ @Test
+ public void testShreddedBinaryBoundsCounts() throws IOException {
+ // counts mode drops shredded bounds
+ byte[] bytes = new byte[20];
+ for (int i = 0; i < bytes.length; i += 1) {
+ bytes[i] = (byte) (i + 1);
+ }
Review Comment:
Will clean up
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
