hcrosse commented on code in PR #2677:
URL: https://github.com/apache/iceberg-go/pull/2677#discussion_r4646983969
##########
table/internal/parquet_files.go:
##########
@@ -2314,6 +2314,71 @@ func (w wrapPqArrowReader) PrunedSchema(projectedIDs
map[int]struct{}, mapping i
return pruneParquetColumns(w.Manifest, projectedIDs, false, mapping)
}
+// ReadDictionaries rebuilds the schema manifest because pqarrow decides
+// dictionary reads when it builds the manifest, not when it reads. The value
+// receiver is safe: the writes go through the embedded *pqarrow.FileReader.
+func (w wrapPqArrowReader) ReadDictionaries(fieldIDs map[int]struct{}, mapping
iceberg.NameMapping) error {
+ leaves := &dictionaryLeafVisitor{fieldIDs: fieldIDs}
+ if _, err := visitParquetManifest(w.Manifest, leaves,
&iceberg.MappedField{Fields: mapping}); err != nil {
+ return err
+ }
+ if len(leaves.colIndices) == 0 {
+ return nil
+ }
+
+ props := w.Props
+ for _, idx := range leaves.colIndices {
+ props.SetReadDict(idx, true)
+ }
+ meta := w.ParquetReader().MetaData()
+ manifest, err := pqarrow.NewSchemaManifest(meta.Schema,
meta.KeyValueMetadata(), &props)
+ if err != nil {
+ return err
+ }
+ w.Props, w.Manifest = props, manifest
+
+ return nil
+}
Review Comment:
`Props` and `Manifest` are promoted fields of the embedded pointer
`*pqarrow.FileReader` (line 2292), not value fields on `wrapPqArrowReader`.
Assigning `w.Props, w.Manifest` mutates the pointed-to `pqarrow.FileReader`
directly, and matches the value receiver used by the other methods on
`wrapPqArrowReader`.
##########
table/internal/interfaces.go:
##########
@@ -75,6 +75,15 @@ type FileReader interface {
ReadTable(context.Context) (arrow.Table, error)
}
+// DictionaryReader is a FileReader that can return selected string and binary
+// leaf columns as Arrow dictionaries.
+type DictionaryReader interface {
+ // ReadDictionaries requests dictionary reads for the leaves with the
given
+ // field IDs. Call it before PrunedSchema or GetRecords. Readers may
still
+ // return plain arrays when a file's stored Arrow schema requires them.
+ ReadDictionaries(fieldIDs map[int]struct{}, mapping
iceberg.NameMapping) error
+}
Review Comment:
`DictionaryReader` is an optional single-method capability interface
type-asserted on an existing `FileReader` (`table/arrow_scanner.go`), following
standard Go single-method extension interfaces rather than duplicating
`FileReader`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]