gszadovszky commented on code in PR #197:
URL: https://github.com/apache/parquet-format/pull/197#discussion_r1155304277
##########
src/main/thrift/parquet.thrift:
##########
@@ -190,6 +190,60 @@ enum FieldRepetitionType {
/** The field is repeated and can contain 0 or more values */
REPEATED = 2;
}
+ /**
+ * Tracks a histogram of repetition and definition levels for either a page
or column chunk.
+ *
+ * This is useful for:
+ * 1. Estimating the size of the data when materialized in memory
+ * 2. For filter push-down on nulls at various levels of nested structures
and
+ * list lengths.
+ */
+ struct RepetitionDefinitionLevelHistogram {
+ /**
+ * When present there is expected to be one element corresponding to each
repetition (i.e. size=max repetition_level+1)
+ * where each element represents the number of time the repetition level
was observed in the data.
+ *
+ * This value is should not be written if max_repetition_level is 0.
Review Comment:
```suggestion
* This value should not be written if max_repetition_level is 0.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -966,6 +1039,13 @@ struct ColumnIndex {
/** A list containing the number of null values for each page **/
5: optional list<i64> null_counts
+ /**
+ * Repetition and definition level histograms for the page.
Review Comment:
```suggestion
* Repetition and definition level histograms for the pages.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -190,6 +190,60 @@ enum FieldRepetitionType {
/** The field is repeated and can contain 0 or more values */
REPEATED = 2;
}
+ /**
+ * Tracks a histogram of repetition and definition levels for either a page
or column chunk.
+ *
+ * This is useful for:
+ * 1. Estimating the size of the data when materialized in memory
+ * 2. For filter push-down on nulls at various levels of nested structures
and
+ * list lengths.
+ */
+ struct RepetitionDefinitionLevelHistogram {
+ /**
+ * When present there is expected to be one element corresponding to each
repetition (i.e. size=max repetition_level+1)
+ * where each element represents the number of time the repetition level
was observed in the data.
+ *
+ * This value is should not be written if max_repetition_level is 0.
+ **/
+ 1: optional list<i64> repetition_level_histogram;
+ /**
+ * Same as repetition_level_histogram except for definition levels.
+ *
+ * This value is should not be written when max_definition_level is 0.
Review Comment:
```suggestion
* This value should not be written when max_definition_level is 0.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -966,6 +1039,13 @@ struct ColumnIndex {
/** A list containing the number of null values for each page **/
5: optional list<i64> null_counts
+ /**
+ * Repetition and definition level histograms for the page.
+ *
+ * This contains has some redundancy with null_counts, however, to accomdate
+ * the widest range of readers both should be populated.
Review Comment:
```suggestion
* This contains some redundancy with null_counts, however, to accomodate
the
* widest range of readers both should be populated.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -580,7 +648,12 @@ struct DataPageHeaderV2 {
If missing it is considered compressed */
7: optional bool is_compressed = 1;
- /** optional statistics for the data in this page **/
+ /**
+ * optional statistics for the data in this page
+ *
+ * See notes on DataPage struct for use-cases and recommendations on a
discussion on
+ * recommendations around populating this field.
Review Comment:
```suggestion
* See notes on DataPage struct for use-cases and recommendations around
populating
* this field.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
