[
https://issues.apache.org/jira/browse/PARQUET-1211?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16467557#comment-16467557
]
ASF GitHub Bot commented on PARQUET-1211:
-----------------------------------------
zivanfi commented on a change in pull request #456: PARQUET-1211: Write column
indexes: read/write API
URL: https://github.com/apache/parquet-mr/pull/456#discussion_r186768014
##########
File path:
parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
##########
@@ -892,4 +898,60 @@ public void testColumnOrders() throws IOException {
assertEquals(ColumnOrder.undefined(),
columns.get(1).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.undefined(),
columns.get(2).getPrimitiveType().columnOrder());
}
+
+ @Test
+ public void testOffsetIndexConversion() {
+ OffsetIndexBuilder builder = OffsetIndexBuilder.getBuilder();
+ builder.add(1000, 10000, 0);
+ builder.add(22000, 12000, 100);
+ OffsetIndex offsetIndex = ParquetMetadataConverter
+
.fromParquetOffsetIndex(ParquetMetadataConverter.toParquetOffsetIndex(builder.build(100000)));
+ assertEquals(2, offsetIndex.getPageCount());
+ assertEquals(101000, offsetIndex.getOffset(0));
+ assertEquals(10000, offsetIndex.getCompressedPageSize(0));
+ assertEquals(0, offsetIndex.getFirstRowIndex(0));
+ assertEquals(122000, offsetIndex.getOffset(1));
+ assertEquals(12000, offsetIndex.getCompressedPageSize(1));
+ assertEquals(100, offsetIndex.getFirstRowIndex(1));
+ }
+
+ @Test
+ public void testColumnIndexConversion() {
+ PrimitiveType type =
Types.required(PrimitiveTypeName.INT64).named("test_int64");
+ ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type);
+ Statistics<?> stats = Statistics.createStats(type);
+ stats.incrementNumNulls(16);
+ stats.updateStats(-100l);
+ stats.updateStats(100l);
+ builder.add(stats);
+ stats = Statistics.createStats(type);
+ stats.incrementNumNulls(111);
+ builder.add(stats);
+ stats = Statistics.createStats(type);
+ stats.updateStats(200l);
+ stats.updateStats(500l);
+ builder.add(stats);
+ org.apache.parquet.format.ColumnIndex parquetColumnIndex =
+ ParquetMetadataConverter.toParquetColumnIndex(type, builder.build());
+ ColumnIndex columnIndex =
ParquetMetadataConverter.fromParquetColumnIndex(type, parquetColumnIndex);
+ assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
+ assertTrue(Arrays.asList(false, true,
false).equals(columnIndex.getNullPages()));
+ assertTrue(Arrays.asList(16l, 111l,
0l).equals(columnIndex.getNullCounts()));
+ assertTrue(Arrays.asList(
+ ByteBuffer.wrap(BytesUtils.longToBytes(-100l)),
+ ByteBuffer.allocate(0),
+
ByteBuffer.wrap(BytesUtils.longToBytes(200l))).equals(columnIndex.getMinValues()));
+ assertTrue(Arrays.asList(
+ ByteBuffer.wrap(BytesUtils.longToBytes(100l)),
+ ByteBuffer.allocate(0),
+
ByteBuffer.wrap(BytesUtils.longToBytes(500l))).equals(columnIndex.getMaxValues()));
+
+ assertNull("Should handle null column index", ParquetMetadataConverter
+
.toParquetColumnIndex(Types.required(PrimitiveTypeName.INT32).named("test_int32"),
null));
+ assertNull("Should handle unsupported types", ParquetMetadataConverter
Review comment:
Should ignore unsupported types.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Write column indexes: read/write API
> ------------------------------------
>
> Key: PARQUET-1211
> URL: https://issues.apache.org/jira/browse/PARQUET-1211
> Project: Parquet
> Issue Type: Sub-task
> Reporter: Gabor Szadovszky
> Assignee: Gabor Szadovszky
> Priority: Major
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
