This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new c3151ade feat(parquet/schema): Wire up logical type functionality for
writing geo types (#960)
c3151ade is described below
commit c3151ade7001dd4524f6fc5cf6e0e420bb1647c9
Author: Colton Loftus <[email protected]>
AuthorDate: Mon Jul 20 14:28:11 2026 -0400
feat(parquet/schema): Wire up logical type functionality for writing geo
types (#960)
Implement the LogicalType interface for Geometry and Geography so the
Parquet write path emits proper geospatial metadata instead of plain byte-array
blobs.
---
parquet/internal/encoding/encoder.go | 2 -
.../gen-go/parquet/GoUnusedProtection__.go | 2 +-
parquet/internal/gen-go/parquet/parquet-consts.go | 4 +-
parquet/internal/gen-go/parquet/parquet.go | 938 ++++++++++++++++++---
parquet/schema/logical_types.go | 258 ++++++
parquet/schema/logical_types_test.go | 22 +
parquet/schema/schema_element_test.go | 10 +
parquet/variant/basic_type_stringer.go | 8 +-
parquet/variant/primitive_type_stringer.go | 8 +-
parquet/version_string.go | 5 +-
10 files changed, 1110 insertions(+), 147 deletions(-)
diff --git a/parquet/internal/encoding/encoder.go
b/parquet/internal/encoding/encoder.go
index 64a992e7..e6561d69 100644
--- a/parquet/internal/encoding/encoder.go
+++ b/parquet/internal/encoding/encoder.go
@@ -31,8 +31,6 @@ import (
"github.com/apache/arrow-go/v18/parquet/schema"
)
-//go:generate go run ../../../arrow/_tools/tmpl/main.go -i
-data=physical_types.tmpldata plain_encoder_types.gen.go.tmpl
typed_encoder.gen.go.tmpl
-
// EncoderTraits is an interface for the different types to make it more
// convenient to construct encoders for specific types.
type EncoderTraits interface {
diff --git a/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
b/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
index 642f3dc7..2b484bfa 100644
--- a/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
+++ b/parquet/internal/gen-go/parquet/GoUnusedProtection__.go
@@ -1,4 +1,4 @@
-// Code generated by Thrift Compiler (0.21.0). DO NOT EDIT.
+// Code generated by Thrift Compiler (0.22.0). DO NOT EDIT.
package parquet
diff --git a/parquet/internal/gen-go/parquet/parquet-consts.go
b/parquet/internal/gen-go/parquet/parquet-consts.go
index 4843d4af..7e3fe145 100644
--- a/parquet/internal/gen-go/parquet/parquet-consts.go
+++ b/parquet/internal/gen-go/parquet/parquet-consts.go
@@ -1,4 +1,4 @@
-// Code generated by Thrift Compiler (0.21.0). DO NOT EDIT.
+// Code generated by Thrift Compiler (0.22.0). DO NOT EDIT.
package parquet
@@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
+ "iter"
"log/slog"
"time"
thrift "github.com/apache/thrift/lib/go/thrift"
@@ -19,6 +20,7 @@ var _ = bytes.Equal
var _ = context.Background
var _ = errors.New
var _ = fmt.Printf
+var _ = iter.Pull[int]
var _ = slog.Log
var _ = time.Now
var _ = thrift.ZERO
diff --git a/parquet/internal/gen-go/parquet/parquet.go
b/parquet/internal/gen-go/parquet/parquet.go
index db24d390..54547a26 100644
--- a/parquet/internal/gen-go/parquet/parquet.go
+++ b/parquet/internal/gen-go/parquet/parquet.go
@@ -1,4 +1,4 @@
-// Code generated by Thrift Compiler (0.21.0). DO NOT EDIT.
+// Code generated by Thrift Compiler (0.22.0). DO NOT EDIT.
package parquet
@@ -8,6 +8,7 @@ import (
"database/sql/driver"
"errors"
"fmt"
+ "iter"
"log/slog"
"time"
thrift "github.com/apache/thrift/lib/go/thrift"
@@ -20,6 +21,7 @@ var _ = bytes.Equal
var _ = context.Background
var _ = errors.New
var _ = fmt.Printf
+var _ = iter.Pull[int]
var _ = slog.Log
var _ = time.Now
var _ = thrift.ZERO
@@ -32,6 +34,7 @@ var _ = regexp.MatchString
//For example INT16 is not included as a type since a good encoding of INT32
//would handle this.
type Type int64
+
const (
Type_BOOLEAN Type = 0
Type_INT32 Type = 1
@@ -43,6 +46,27 @@ const (
Type_FIXED_LEN_BYTE_ARRAY Type = 7
)
+var knownTypeValues = []Type{
+ Type_BOOLEAN,
+ Type_INT32,
+ Type_INT64,
+ Type_INT96,
+ Type_FLOAT,
+ Type_DOUBLE,
+ Type_BYTE_ARRAY,
+ Type_FIXED_LEN_BYTE_ARRAY,
+}
+
+func TypeValues() iter.Seq[Type] {
+ return func(yield func(Type) bool) {
+ for _, v := range knownTypeValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p Type) String() string {
switch p {
case Type_BOOLEAN: return "BOOLEAN"
@@ -103,11 +127,12 @@ func (p *Type) Value() (driver.Value, error) {
return int64(*p), nil
}
-//DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet.
+//DEPRECATED: Common types used by frameworks (e.g. Hive, Pig) using parquet.
//ConvertedType is superseded by LogicalType. This enum should not be
extended.
//
//See LogicalTypes.md for conversion between ConvertedType and LogicalType.
type ConvertedType int64
+
const (
ConvertedType_UTF8 ConvertedType = 0
ConvertedType_MAP ConvertedType = 1
@@ -133,6 +158,41 @@ const (
ConvertedType_INTERVAL ConvertedType = 21
)
+var knownConvertedTypeValues = []ConvertedType{
+ ConvertedType_UTF8,
+ ConvertedType_MAP,
+ ConvertedType_MAP_KEY_VALUE,
+ ConvertedType_LIST,
+ ConvertedType_ENUM,
+ ConvertedType_DECIMAL,
+ ConvertedType_DATE,
+ ConvertedType_TIME_MILLIS,
+ ConvertedType_TIME_MICROS,
+ ConvertedType_TIMESTAMP_MILLIS,
+ ConvertedType_TIMESTAMP_MICROS,
+ ConvertedType_UINT_8,
+ ConvertedType_UINT_16,
+ ConvertedType_UINT_32,
+ ConvertedType_UINT_64,
+ ConvertedType_INT_8,
+ ConvertedType_INT_16,
+ ConvertedType_INT_32,
+ ConvertedType_INT_64,
+ ConvertedType_JSON,
+ ConvertedType_BSON,
+ ConvertedType_INTERVAL,
+}
+
+func ConvertedTypeValues() iter.Seq[ConvertedType] {
+ return func(yield func(ConvertedType) bool) {
+ for _, v := range knownConvertedTypeValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p ConvertedType) String() string {
switch p {
case ConvertedType_UTF8: return "UTF8"
@@ -223,12 +283,29 @@ func (p *ConvertedType) Value() (driver.Value, error) {
//Representation of Schemas
type FieldRepetitionType int64
+
const (
FieldRepetitionType_REQUIRED FieldRepetitionType = 0
FieldRepetitionType_OPTIONAL FieldRepetitionType = 1
FieldRepetitionType_REPEATED FieldRepetitionType = 2
)
+var knownFieldRepetitionTypeValues = []FieldRepetitionType{
+ FieldRepetitionType_REQUIRED,
+ FieldRepetitionType_OPTIONAL,
+ FieldRepetitionType_REPEATED,
+}
+
+func FieldRepetitionTypeValues() iter.Seq[FieldRepetitionType] {
+ return func(yield func(FieldRepetitionType) bool) {
+ for _, v := range knownFieldRepetitionTypeValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p FieldRepetitionType) String() string {
switch p {
case FieldRepetitionType_REQUIRED: return "REQUIRED"
@@ -281,6 +358,7 @@ func (p *FieldRepetitionType) Value() (driver.Value, error)
{
//Edge interpolation algorithm for Geography logical type
type EdgeInterpolationAlgorithm int64
+
const (
EdgeInterpolationAlgorithm_SPHERICAL EdgeInterpolationAlgorithm = 0
EdgeInterpolationAlgorithm_VINCENTY EdgeInterpolationAlgorithm = 1
@@ -289,6 +367,24 @@ const (
EdgeInterpolationAlgorithm_KARNEY EdgeInterpolationAlgorithm = 4
)
+var knownEdgeInterpolationAlgorithmValues = []EdgeInterpolationAlgorithm{
+ EdgeInterpolationAlgorithm_SPHERICAL,
+ EdgeInterpolationAlgorithm_VINCENTY,
+ EdgeInterpolationAlgorithm_THOMAS,
+ EdgeInterpolationAlgorithm_ANDOYER,
+ EdgeInterpolationAlgorithm_KARNEY,
+}
+
+func EdgeInterpolationAlgorithmValues() iter.Seq[EdgeInterpolationAlgorithm] {
+ return func(yield func(EdgeInterpolationAlgorithm) bool) {
+ for _, v := range knownEdgeInterpolationAlgorithmValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p EdgeInterpolationAlgorithm) String() string {
switch p {
case EdgeInterpolationAlgorithm_SPHERICAL: return "SPHERICAL"
@@ -347,6 +443,7 @@ func (p *EdgeInterpolationAlgorithm) Value() (driver.Value,
error) {
//enums are also used to specify the encoding of definition and repetition
levels.
//See the accompanying doc for the details of the more complicated encodings.
type Encoding int64
+
const (
Encoding_PLAIN Encoding = 0
Encoding_PLAIN_DICTIONARY Encoding = 2
@@ -359,6 +456,28 @@ const (
Encoding_BYTE_STREAM_SPLIT Encoding = 9
)
+var knownEncodingValues = []Encoding{
+ Encoding_PLAIN,
+ Encoding_PLAIN_DICTIONARY,
+ Encoding_RLE,
+ Encoding_BIT_PACKED,
+ Encoding_DELTA_BINARY_PACKED,
+ Encoding_DELTA_LENGTH_BYTE_ARRAY,
+ Encoding_DELTA_BYTE_ARRAY,
+ Encoding_RLE_DICTIONARY,
+ Encoding_BYTE_STREAM_SPLIT,
+}
+
+func EncodingValues() iter.Seq[Encoding] {
+ return func(yield func(Encoding) bool) {
+ for _, v := range knownEncodingValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p Encoding) String() string {
switch p {
case Encoding_PLAIN: return "PLAIN"
@@ -429,6 +548,7 @@ func (p *Encoding) Value() (driver.Value, error) {
//
//See Compression.md for a detailed specification of these algorithms.
type CompressionCodec int64
+
const (
CompressionCodec_UNCOMPRESSED CompressionCodec = 0
CompressionCodec_SNAPPY CompressionCodec = 1
@@ -440,6 +560,27 @@ const (
CompressionCodec_LZ4_RAW CompressionCodec = 7
)
+var knownCompressionCodecValues = []CompressionCodec{
+ CompressionCodec_UNCOMPRESSED,
+ CompressionCodec_SNAPPY,
+ CompressionCodec_GZIP,
+ CompressionCodec_LZO,
+ CompressionCodec_BROTLI,
+ CompressionCodec_LZ4,
+ CompressionCodec_ZSTD,
+ CompressionCodec_LZ4_RAW,
+}
+
+func CompressionCodecValues() iter.Seq[CompressionCodec] {
+ return func(yield func(CompressionCodec) bool) {
+ for _, v := range knownCompressionCodecValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p CompressionCodec) String() string {
switch p {
case CompressionCodec_UNCOMPRESSED: return "UNCOMPRESSED"
@@ -501,6 +642,7 @@ func (p *CompressionCodec) Value() (driver.Value, error) {
}
type PageType int64
+
const (
PageType_DATA_PAGE PageType = 0
PageType_INDEX_PAGE PageType = 1
@@ -508,6 +650,23 @@ const (
PageType_DATA_PAGE_V2 PageType = 3
)
+var knownPageTypeValues = []PageType{
+ PageType_DATA_PAGE,
+ PageType_INDEX_PAGE,
+ PageType_DICTIONARY_PAGE,
+ PageType_DATA_PAGE_V2,
+}
+
+func PageTypeValues() iter.Seq[PageType] {
+ return func(yield func(PageType) bool) {
+ for _, v := range knownPageTypeValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p PageType) String() string {
switch p {
case PageType_DATA_PAGE: return "DATA_PAGE"
@@ -563,12 +722,29 @@ func (p *PageType) Value() (driver.Value, error) {
//Enum to annotate whether lists of min/max elements inside ColumnIndex
//are ordered and if so, in which direction.
type BoundaryOrder int64
+
const (
BoundaryOrder_UNORDERED BoundaryOrder = 0
BoundaryOrder_ASCENDING BoundaryOrder = 1
BoundaryOrder_DESCENDING BoundaryOrder = 2
)
+var knownBoundaryOrderValues = []BoundaryOrder{
+ BoundaryOrder_UNORDERED,
+ BoundaryOrder_ASCENDING,
+ BoundaryOrder_DESCENDING,
+}
+
+func BoundaryOrderValues() iter.Seq[BoundaryOrder] {
+ return func(yield func(BoundaryOrder) bool) {
+ for _, v := range knownBoundaryOrderValues {
+ if !yield(v) {
+ return
+ }
+ }
+ }
+}
+
func (p BoundaryOrder) String() string {
switch p {
case BoundaryOrder_UNORDERED: return "UNORDERED"
@@ -1162,16 +1338,16 @@ func (p *BoundingBox) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetXmin{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Xmin is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Xmin is not set"))
}
if !issetXmax{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Xmax is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Xmax is not set"))
}
if !issetYmin{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Ymin is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Ymin is not set"))
}
if !issetYmax{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Ymax is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Ymax is not set"))
}
return nil
}
@@ -1693,6 +1869,10 @@ func (p *GeospatialStatistics) Validate() error {
// - MinValue
// - IsMaxValueExact: If true, max_value is the actual maximum value for a
column
// - IsMinValueExact: If true, min_value is the actual minimum value for a
column
+// - NanCount: Count of NaN values in the column; only present if physical
type is FLOAT
+// or DOUBLE, or logical type is FLOAT16.
+// If this field is not present, readers MUST assume NaNs may be present
+// (i.e. MUST assume nan_count > 0 and MAY NOT assume nan_count == 0).
//
type Statistics struct {
Max []byte `thrift:"max,1" db:"max" json:"max,omitempty"`
@@ -1703,6 +1883,7 @@ type Statistics struct {
MinValue []byte `thrift:"min_value,6" db:"min_value"
json:"min_value,omitempty"`
IsMaxValueExact *bool `thrift:"is_max_value_exact,7"
db:"is_max_value_exact" json:"is_max_value_exact,omitempty"`
IsMinValueExact *bool `thrift:"is_min_value_exact,8"
db:"is_min_value_exact" json:"is_min_value_exact,omitempty"`
+ NanCount *int64 `thrift:"nan_count,9" db:"nan_count"
json:"nan_count,omitempty"`
}
func NewStatistics() *Statistics {
@@ -1773,6 +1954,15 @@ func (p *Statistics) GetIsMinValueExact() bool {
return *p.IsMinValueExact
}
+var Statistics_NanCount_DEFAULT int64
+
+func (p *Statistics) GetNanCount() int64 {
+ if !p.IsSetNanCount() {
+ return Statistics_NanCount_DEFAULT
+ }
+ return *p.NanCount
+}
+
func (p *Statistics) IsSetMax() bool {
return p.Max != nil
}
@@ -1805,6 +1995,10 @@ func (p *Statistics) IsSetIsMinValueExact() bool {
return p.IsMinValueExact != nil
}
+func (p *Statistics) IsSetNanCount() bool {
+ return p.NanCount != nil
+}
+
func (p *Statistics) Read(ctx context.Context, iprot thrift.TProtocol) error {
if _, err := iprot.ReadStructBegin(ctx); err != nil {
return thrift.PrependError(fmt.Sprintf("%T read error: ", p),
err)
@@ -1900,6 +2094,16 @@ func (p *Statistics) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return err
}
}
+ case 9:
+ if fieldTypeId == thrift.I64 {
+ if err := p.ReadField9(ctx, iprot); err != nil {
+ return err
+ }
+ } else {
+ if err := iprot.Skip(ctx, fieldTypeId); err !=
nil {
+ return err
+ }
+ }
default:
if err := iprot.Skip(ctx, fieldTypeId); err != nil {
return err
@@ -1987,6 +2191,15 @@ func (p *Statistics) ReadField8(ctx context.Context,
iprot thrift.TProtocol) err
return nil
}
+func (p *Statistics) ReadField9(ctx context.Context, iprot thrift.TProtocol)
error {
+ if v, err := iprot.ReadI64(ctx); err != nil {
+ return thrift.PrependError("error reading field 9: ", err)
+ } else {
+ p.NanCount = &v
+ }
+ return nil
+}
+
func (p *Statistics) Write(ctx context.Context, oprot thrift.TProtocol) error {
if err := oprot.WriteStructBegin(ctx, "Statistics"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -2000,6 +2213,7 @@ func (p *Statistics) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if err := p.writeField6(ctx, oprot); err != nil { return err }
if err := p.writeField7(ctx, oprot); err != nil { return err }
if err := p.writeField8(ctx, oprot); err != nil { return err }
+ if err := p.writeField9(ctx, oprot); err != nil { return err }
}
if err := oprot.WriteFieldStop(ctx); err != nil {
return thrift.PrependError("write field stop error: ", err)
@@ -2130,6 +2344,21 @@ func (p *Statistics) writeField8(ctx context.Context,
oprot thrift.TProtocol) (e
return err
}
+func (p *Statistics) writeField9(ctx context.Context, oprot thrift.TProtocol)
(err error) {
+ if p.IsSetNanCount() {
+ if err := oprot.WriteFieldBegin(ctx, "nan_count", thrift.I64,
9); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
begin error 9:nan_count: ", p), err)
+ }
+ if err := oprot.WriteI64(ctx, int64(*p.NanCount)); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T.nan_count
(9) field write error: ", p), err)
+ }
+ if err := oprot.WriteFieldEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
end error 9:nan_count: ", p), err)
+ }
+ }
+ return err
+}
+
func (p *Statistics) Equals(other *Statistics) bool {
if p == other {
return true
@@ -2164,6 +2393,12 @@ func (p *Statistics) Equals(other *Statistics) bool {
}
if (*p.IsMinValueExact) != (*other.IsMinValueExact) { return
false }
}
+ if p.NanCount != other.NanCount {
+ if p.NanCount == nil || other.NanCount == nil {
+ return false
+ }
+ if (*p.NanCount) != (*other.NanCount) { return false }
+ }
return true
}
@@ -2940,10 +3175,10 @@ func (p *DecimalType) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetScale{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Scale is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Scale is not set"))
}
if !issetPrecision{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Precision is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Precision is not set"))
}
return nil
}
@@ -3446,7 +3681,7 @@ func (p *TimeUnit) ReadField3(ctx context.Context, iprot
thrift.TProtocol) error
func (p *TimeUnit) Write(ctx context.Context, oprot thrift.TProtocol) error {
if c := p.CountSetFieldsTimeUnit(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "TimeUnit"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -3634,10 +3869,10 @@ func (p *TimestampType) Read(ctx context.Context, iprot
thrift.TProtocol) error
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetIsAdjustedToUTC{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsAdjustedToUTC is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsAdjustedToUTC is not set"))
}
if !issetUnit{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Unit is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Unit is not set"))
}
return nil
}
@@ -3825,10 +4060,10 @@ func (p *TimeType) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetIsAdjustedToUTC{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsAdjustedToUTC is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsAdjustedToUTC is not set"))
}
if !issetUnit{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Unit is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Unit is not set"))
}
return nil
}
@@ -4011,10 +4246,10 @@ func (p *IntType) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetBitWidth{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field BitWidth is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field BitWidth is not set"))
}
if !issetIsSigned{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsSigned is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field IsSigned is not set"))
}
return nil
}
@@ -4434,7 +4669,7 @@ func (p *VariantType) Validate() error {
// Embedded Geometry logical type annotation
//
-// Geospatial features in the Well-Known Binary (WKB) format and edges
interpolation
+// Geospatial features in the Well-Known Binary (WKB) format and `edges`
interpolation
// is always linear/planar.
//
// A custom CRS can be set by the crs field. If unset, it defaults to
"OGC:CRS84",
@@ -4591,13 +4826,13 @@ func (p *GeometryType) Validate() error {
// Embedded Geography logical type annotation
//
// Geospatial features in the WKB format with an explicit
(non-linear/non-planar)
-// edges interpolation algorithm.
+// `edges` interpolation algorithm.
//
// A custom geographic CRS can be set by the crs field, where longitudes are
// bound by [-180, 180] and latitudes are bound by [-90, 90]. If unset, the CRS
// defaults to "OGC:CRS84".
//
-// An optional algorithm can be set to correctly interpret edges interpolation
+// An optional algorithm can be set to correctly interpret `edges`
interpolation
// of the geometries. If unset, the algorithm defaults to SPHERICAL.
//
// Allowed for physical type: BYTE_ARRAY.
@@ -5470,7 +5705,7 @@ func (p *LogicalType) ReadField18(ctx context.Context,
iprot thrift.TProtocol) e
func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error
{
if c := p.CountSetFieldsLogicalType(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "LogicalType"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -5808,7 +6043,7 @@ func (p *LogicalType) Validate() error {
return nil
}
-// Represents a element inside a schema definition.
+// Represents an element inside a schema definition.
// - if it is a group (inner node) then type is undefined and num_children is
defined
// - if it is a primitive type (leaf) then type is defined and num_children
is undefined
// the nodes are listed in depth first traversal order.
@@ -6112,7 +6347,7 @@ func (p *SchemaElement) Read(ctx context.Context, iprot
thrift.TProtocol) error
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetName{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Name is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Name is not set"))
}
return nil
}
@@ -6470,7 +6705,7 @@ func (p *SchemaElement) Validate() error {
// Attributes:
// - NumValues: Number of values, including NULLs, in this data page.
//
-// If a OffsetIndex is present, a page must begin at a row
+// If an OffsetIndex is present, a page must begin at a row
// boundary (repetition_level = 0). Otherwise, pages may begin
// within a row (repetition_level > 0).
//
@@ -6614,16 +6849,16 @@ func (p *DataPageHeader) Read(ctx context.Context,
iprot thrift.TProtocol) error
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetNumValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"))
}
if !issetEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"))
}
if !issetDefinitionLevelEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DefinitionLevelEncoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DefinitionLevelEncoding is not set"))
}
if !issetRepetitionLevelEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RepetitionLevelEncoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RepetitionLevelEncoding is not set"))
}
return nil
}
@@ -6989,10 +7224,10 @@ func (p *DictionaryPageHeader) Read(ctx
context.Context, iprot thrift.TProtocol)
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetNumValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"))
}
if !issetEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"))
}
return nil
}
@@ -7125,10 +7360,15 @@ func (p *DictionaryPageHeader) Validate() error {
return nil
}
-// New page format allowing reading levels without decompressing the data
+// Alternate page format allowing reading levels without decompressing the data
// Repetition and definition levels are uncompressed
// The remaining section containing the data is compressed if is_compressed is
true
//
+// Implementation note - this header is not necessarily a strict improvement
over
+// `DataPageHeader` (in particular the original header might provide better
compression
+// in some scenarios). Page indexes require pages to start and end at row
boundaries,
+// regardless of which page header is used.
+//
//
// Attributes:
// - NumValues: Number of values, including NULLs, in this data page. *
@@ -7143,7 +7383,7 @@ func (p *DictionaryPageHeader) Validate() error {
// - RepetitionLevelsByteLength: Length of the repetition levels
// - IsCompressed: Whether the values are compressed.
// Which means the section of the page between
-// definition_levels_byte_length + repetition_levels_byte_length + 1 and
compressed_page_size (included)
+// definition_levels_byte_length + repetition_levels_byte_length and
compressed_page_size (included)
// is compressed with the compression_codec.
// If missing it is considered compressed
// - Statistics: Optional statistics for the data in this page *
@@ -7345,22 +7585,22 @@ func (p *DataPageHeaderV2) Read(ctx context.Context,
iprot thrift.TProtocol) err
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetNumValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"))
}
if !issetNumNulls{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumNulls is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumNulls is not set"))
}
if !issetNumRows{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"))
}
if !issetEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"))
}
if !issetDefinitionLevelsByteLength{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DefinitionLevelsByteLength is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DefinitionLevelsByteLength is not set"))
}
if !issetRepetitionLevelsByteLength{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RepetitionLevelsByteLength is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RepetitionLevelsByteLength is not set"))
}
return nil
}
@@ -7777,7 +8017,7 @@ func (p *BloomFilterAlgorithm) ReadField1(ctx
context.Context, iprot thrift.TPro
func (p *BloomFilterAlgorithm) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if c := p.CountSetFieldsBloomFilterAlgorithm(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "BloomFilterAlgorithm"); err !=
nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -8015,7 +8255,7 @@ func (p *BloomFilterHash) ReadField1(ctx context.Context,
iprot thrift.TProtocol
func (p *BloomFilterHash) Write(ctx context.Context, oprot thrift.TProtocol)
error {
if c := p.CountSetFieldsBloomFilterHash(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "BloomFilterHash"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -8248,7 +8488,7 @@ func (p *BloomFilterCompression) ReadField1(ctx
context.Context, iprot thrift.TP
func (p *BloomFilterCompression) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if c := p.CountSetFieldsBloomFilterCompression(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "BloomFilterCompression"); err !=
nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -8456,16 +8696,16 @@ func (p *BloomFilterHeader) Read(ctx context.Context,
iprot thrift.TProtocol) er
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetNumBytes{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumBytes is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumBytes is not set"))
}
if !issetAlgorithm{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Algorithm is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Algorithm is not set"))
}
if !issetHash{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Hash is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Hash is not set"))
}
if !issetCompression{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Compression is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Compression is not set"))
}
return nil
}
@@ -8615,10 +8855,10 @@ func (p *BloomFilterHeader) Validate() error {
// - Type: the type of the page: indicates which of the *_header fields is
set *
// - UncompressedPageSize: Uncompressed page size in bytes (not including
this header) *
// - CompressedPageSize: Compressed (and potentially encrypted) page size in
bytes, not including this header *
-// - Crc: The 32-bit CRC checksum for the page, to be be calculated as
follows:
+// - Crc: The 32-bit CRC checksum for the page, to be calculated as follows:
//
// - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7,
-// the same as in e.g. GZip).
+// the same as in e.g. GZIP).
// - All page types can have a CRC (v1 and v2 data pages, dictionary pages,
// etc.).
// - The CRC is computed on the serialization binary representation of the page
@@ -8848,13 +9088,13 @@ func (p *PageHeader) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetType{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Type is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Type is not set"))
}
if !issetUncompressedPageSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field UncompressedPageSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field UncompressedPageSize is not set"))
}
if !issetCompressedPageSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field CompressedPageSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field CompressedPageSize is not set"))
}
return nil
}
@@ -9197,7 +9437,7 @@ func (p *KeyValue) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetKey{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Key is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Key is not set"))
}
return nil
}
@@ -9405,13 +9645,13 @@ func (p *SortingColumn) Read(ctx context.Context, iprot
thrift.TProtocol) error
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetColumnIdx{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field ColumnIdx is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field ColumnIdx is not set"))
}
if !issetDescending{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Descending is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Descending is not set"))
}
if !issetNullsFirst{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NullsFirst is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NullsFirst is not set"))
}
return nil
}
@@ -9635,13 +9875,13 @@ func (p *PageEncodingStats) Read(ctx context.Context,
iprot thrift.TProtocol) er
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetPageType{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PageType is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PageType is not set"))
}
if !issetEncoding{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encoding is not set"))
}
if !issetCount{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Count is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Count is not set"))
}
return nil
}
@@ -10199,28 +10439,28 @@ func (p *ColumnMetaData) Read(ctx context.Context,
iprot thrift.TProtocol) error
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetType{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Type is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Type is not set"))
}
if !issetEncodings{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encodings is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Encodings is not set"))
}
if !issetPathInSchema{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PathInSchema is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PathInSchema is not set"))
}
if !issetCodec{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Codec is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Codec is not set"))
}
if !issetNumValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumValues is not set"))
}
if !issetTotalUncompressedSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalUncompressedSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalUncompressedSize is not set"))
}
if !issetTotalCompressedSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalCompressedSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalCompressedSize is not set"))
}
if !issetDataPageOffset{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DataPageOffset is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field DataPageOffset is not set"))
}
return nil
}
@@ -10977,7 +11217,7 @@ func (p *EncryptionWithColumnKey) Read(ctx
context.Context, iprot thrift.TProtoc
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetPathInSchema{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PathInSchema is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PathInSchema is not set"))
}
return nil
}
@@ -11224,7 +11464,7 @@ func (p *ColumnCryptoMetaData) ReadField2(ctx
context.Context, iprot thrift.TPro
func (p *ColumnCryptoMetaData) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if c := p.CountSetFieldsColumnCryptoMetaData(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "ColumnCryptoMetaData"); err !=
nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -11311,7 +11551,22 @@ func (p *ColumnCryptoMetaData) Validate() error {
// - FilePath: File where column data is stored. If not set, assumed to be
same file as
// metadata. This path is relative to the current file.
//
-// - FileOffset: Deprecated: Byte offset in file_path to the ColumnMetaData
+// As of December 2025, the only known use-case for this field is writing
summary
+// parquet files (i.e. "_metadata" files). These files consolidate footers
from
+// multiple parquet files to allow for efficient reading of footers to avoid
file
+// listing costs and prune out files that do not need to be read based on
statistics.
+//
+// These files do not appear to have ever been formally specified in the
specification.
+// and are potentially problematic from a correctness perspective [1].
+//
+// [1] https://lists.apache.org/thread/ootf2kmyg3p01b1bvplpvp4ftd1bt72d
+//
+// There is no other known usage of this field. Specifically, there are no
known
+// reference implementations that will read externally stored column data if
this field is populated
+// within a standard parquet file. Making use of the field for this purpose is
+// not considered part of the Parquet specification.
+//
+// - FileOffset: DEPRECATED: Byte offset in file_path to the ColumnMetaData
//
// Past use of this field has been inconsistent, with some implementations
// using it to point to the ColumnMetaData and some using it to point to
@@ -11575,7 +11830,7 @@ func (p *ColumnChunk) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetFileOffset{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field FileOffset is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field FileOffset is not set"))
}
return nil
}
@@ -12083,13 +12338,13 @@ func (p *RowGroup) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetColumns{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Columns is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Columns is not set"))
}
if !issetTotalByteSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalByteSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field TotalByteSize is not set"))
}
if !issetNumRows{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"))
}
return nil
}
@@ -12462,6 +12717,172 @@ func (p *TypeDefinedOrder) Validate() error {
return nil
}
+// Empty struct to signal IEEE 754 total order for floating point types
+type IEEE754TotalOrder struct {
+}
+
+func NewIEEE754TotalOrder() *IEEE754TotalOrder {
+ return &IEEE754TotalOrder{}
+}
+
+func (p *IEEE754TotalOrder) Read(ctx context.Context, iprot thrift.TProtocol)
error {
+ if _, err := iprot.ReadStructBegin(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T read error: ", p),
err)
+ }
+
+
+ for {
+ _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx)
+ if err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T field %d
read error: ", p, fieldId), err)
+ }
+ if fieldTypeId == thrift.STOP {
+ break
+ }
+ if err := iprot.Skip(ctx, fieldTypeId); err != nil {
+ return err
+ }
+ if err := iprot.ReadFieldEnd(ctx); err != nil {
+ return err
+ }
+ }
+ if err := iprot.ReadStructEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
+ }
+ return nil
+}
+
+func (p *IEEE754TotalOrder) Write(ctx context.Context, oprot thrift.TProtocol)
error {
+ if err := oprot.WriteStructBegin(ctx, "IEEE754TotalOrder"); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
+ }
+ if p != nil {
+ }
+ if err := oprot.WriteFieldStop(ctx); err != nil {
+ return thrift.PrependError("write field stop error: ", err)
+ }
+ if err := oprot.WriteStructEnd(ctx); err != nil {
+ return thrift.PrependError("write struct stop error: ", err)
+ }
+ return nil
+}
+
+func (p *IEEE754TotalOrder) Equals(other *IEEE754TotalOrder) bool {
+ if p == other {
+ return true
+ } else if p == nil || other == nil {
+ return false
+ }
+ return true
+}
+
+func (p *IEEE754TotalOrder) String() string {
+ if p == nil {
+ return "<nil>"
+ }
+ return fmt.Sprintf("IEEE754TotalOrder(%+v)", *p)
+}
+
+func (p *IEEE754TotalOrder) LogValue() slog.Value {
+ if p == nil {
+ return slog.AnyValue(nil)
+ }
+ v := thrift.SlogTStructWrapper{
+ Type: "*parquet.IEEE754TotalOrder",
+ Value: p,
+ }
+ return slog.AnyValue(v)
+}
+
+var _ slog.LogValuer = (*IEEE754TotalOrder)(nil)
+
+func (p *IEEE754TotalOrder) Validate() error {
+ return nil
+}
+
+// Empty struct to signal chronological ordering of physical type INT96
+type Int96TimestampOrder struct {
+}
+
+func NewInt96TimestampOrder() *Int96TimestampOrder {
+ return &Int96TimestampOrder{}
+}
+
+func (p *Int96TimestampOrder) Read(ctx context.Context, iprot
thrift.TProtocol) error {
+ if _, err := iprot.ReadStructBegin(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T read error: ", p),
err)
+ }
+
+
+ for {
+ _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx)
+ if err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T field %d
read error: ", p, fieldId), err)
+ }
+ if fieldTypeId == thrift.STOP {
+ break
+ }
+ if err := iprot.Skip(ctx, fieldTypeId); err != nil {
+ return err
+ }
+ if err := iprot.ReadFieldEnd(ctx); err != nil {
+ return err
+ }
+ }
+ if err := iprot.ReadStructEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
+ }
+ return nil
+}
+
+func (p *Int96TimestampOrder) Write(ctx context.Context, oprot
thrift.TProtocol) error {
+ if err := oprot.WriteStructBegin(ctx, "Int96TimestampOrder"); err !=
nil {
+ return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
+ }
+ if p != nil {
+ }
+ if err := oprot.WriteFieldStop(ctx); err != nil {
+ return thrift.PrependError("write field stop error: ", err)
+ }
+ if err := oprot.WriteStructEnd(ctx); err != nil {
+ return thrift.PrependError("write struct stop error: ", err)
+ }
+ return nil
+}
+
+func (p *Int96TimestampOrder) Equals(other *Int96TimestampOrder) bool {
+ if p == other {
+ return true
+ } else if p == nil || other == nil {
+ return false
+ }
+ return true
+}
+
+func (p *Int96TimestampOrder) String() string {
+ if p == nil {
+ return "<nil>"
+ }
+ return fmt.Sprintf("Int96TimestampOrder(%+v)", *p)
+}
+
+func (p *Int96TimestampOrder) LogValue() slog.Value {
+ if p == nil {
+ return slog.AnyValue(nil)
+ }
+ v := thrift.SlogTStructWrapper{
+ Type: "*parquet.Int96TimestampOrder",
+ Value: p,
+ }
+ return slog.AnyValue(v)
+}
+
+var _ slog.LogValuer = (*Int96TimestampOrder)(nil)
+
+func (p *Int96TimestampOrder) Validate() error {
+ return nil
+}
+
// Union to specify the order used for the min_value and max_value fields for a
// column. This union takes the role of an enhanced enum that allows rich
// elements (which will be needed for a collation-based ordering in the
future).
@@ -12469,6 +12890,9 @@ func (p *TypeDefinedOrder) Validate() error {
// Possible values are:
// * TypeDefinedOrder - the column uses the order defined by its logical or
// physical type (if there is no logical type).
+// * IEEE754TotalOrder - the floating point column uses IEEE 754 total order.
+//
+// * Int96TimestampOrder - the INT96 column uses chronological timestamp order.
//
// If the reader does not support the value of this union, min and max stats
// for this column should be ignored.
@@ -12486,6 +12910,7 @@ func (p *TypeDefinedOrder) Validate() error {
// UINT64 - unsigned comparison
// DECIMAL - signed comparison of the represented value
// DATE - signed comparison
+// FLOAT16 - signed comparison of the represented value (*)
// TIME_MILLIS - signed comparison
// TIME_MICROS - signed comparison
// TIMESTAMP_MILLIS - signed comparison
@@ -12504,30 +12929,73 @@ func (p *TypeDefinedOrder) Validate() error {
// BOOLEAN - false, true
// INT32 - signed comparison
// INT64 - signed comparison
-// INT96 (only used for legacy timestamps) - undefined
+// INT96 (only used for legacy timestamps) - depends on sort order (+)
// FLOAT - signed comparison of the represented value (*)
// DOUBLE - signed comparison of the represented value (*)
// BYTE_ARRAY - unsigned byte-wise comparison
// FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
//
-// (*) Because the sorting order is not specified properly for floating
-// point values (relations vs. total ordering) the following
+// (+) While the INT96 type has been deprecated, at the time of writing it is
+// still used in many legacy systems. It is optional for writers to emit
+// statistics for INT96 columns. Writers that emit stats for such columns
+// should use the INT96_TIMESTAMP_ORDER for this type and order the values
+// according to the legacy rules:
+// - compare the last 4 bytes (days) as a little-endian 32-bit signed
integer
+// - if equal last 4 bytes, compare the first 8 bytes as a little-endian
+// 64-bit signed integer (nanos)
+// If TYPE_ORDER is used for an INT96 column, readers should ignore all
statistics
+// (`min`/`max` fields in `Statistics` and `min_values`/`max_values`
fields in
+// `ColumnIndex`) for that column.
+//
+// (*) Because TYPE_ORDER is ambiguous for floating point types due to
+// underspecified handling of NaN and -0/+0, it is recommended that writers
+// use IEEE_754_TOTAL_ORDER for these types.
+//
+// If TYPE_ORDER is used for floating point types, then the following
// compatibility rules should be applied when reading statistics:
// - If the min is a NaN, it should be ignored.
// - If the max is a NaN, it should be ignored.
+// - If the nan_count field is set, a reader can compute
+// nan_count + null_count == num_values to deduce whether all non-null
+// values are NaN.
// - If the min is +0, the row group may contain -0 values as well.
// - If the max is -0, the row group may contain +0 values as well.
// - When looking for NaN values, min and max should be ignored.
-//
-// When writing statistics the following rules should be followed:
-// - NaNs should not be written to min or max statistics fields.
+// If the nan_count field is set, it can be used to check whether
+// NaNs are present.
+//
+// When writing page or column chunk statistics for columns with
+// TYPE_ORDER order, the following rules must be followed:
+// - The nan_count field must be set for floating point types, even if
+// it is zero.
+// - If the nan_count field is set, min and max statistics fields, when
+// present, must not contain NaN values and must be computed from
+// non-NaN values only. This signals to readers that the min and max
+// statistics are reliable for non-NaN values.
+// - If all non-null values are NaN, min and max statistics must not be
+// written.
// - If the computed max value is zero (whether negative or positive),
// `+0.0` should be written into the max statistics field.
// - If the computed min value is zero (whether negative or positive),
// `-0.0` should be written into the min statistics field.
//
+// When writing column indexes for columns with TYPE_ORDER order, the
+// following rules must be followed:
+// - NaNs must not be written to min_values or max_values.
+// - If all non-null values of a page are NaN, a column index must not
+// be written for this column chunk because min_values and max_values
+// are required.
+// - If the computed max value is zero (whether negative or positive),
+// `+0.0` should be written into the corresponding max_values entry.
+// - If the computed min value is zero (whether negative or positive),
+// `-0.0` should be written into the corresponding min_values entry.
+// - IEEE_754_TOTAL_ORDER
+// - INT96_TIMESTAMP_ORDER
+//
type ColumnOrder struct {
TYPE_ORDER *TypeDefinedOrder `thrift:"TYPE_ORDER,1" db:"TYPE_ORDER"
json:"TYPE_ORDER,omitempty"`
+ IEEE_754_TOTAL_ORDER *IEEE754TotalOrder
`thrift:"IEEE_754_TOTAL_ORDER,2" db:"IEEE_754_TOTAL_ORDER"
json:"IEEE_754_TOTAL_ORDER,omitempty"`
+ INT96_TIMESTAMP_ORDER *Int96TimestampOrder
`thrift:"INT96_TIMESTAMP_ORDER,3" db:"INT96_TIMESTAMP_ORDER"
json:"INT96_TIMESTAMP_ORDER,omitempty"`
}
func NewColumnOrder() *ColumnOrder {
@@ -12543,11 +13011,35 @@ func (p *ColumnOrder) GetTYPE_ORDER()
*TypeDefinedOrder {
return p.TYPE_ORDER
}
+var ColumnOrder_IEEE_754_TOTAL_ORDER_DEFAULT *IEEE754TotalOrder
+
+func (p *ColumnOrder) GetIEEE_754_TOTAL_ORDER() *IEEE754TotalOrder {
+ if !p.IsSetIEEE_754_TOTAL_ORDER() {
+ return ColumnOrder_IEEE_754_TOTAL_ORDER_DEFAULT
+ }
+ return p.IEEE_754_TOTAL_ORDER
+}
+
+var ColumnOrder_INT96_TIMESTAMP_ORDER_DEFAULT *Int96TimestampOrder
+
+func (p *ColumnOrder) GetINT96_TIMESTAMP_ORDER() *Int96TimestampOrder {
+ if !p.IsSetINT96_TIMESTAMP_ORDER() {
+ return ColumnOrder_INT96_TIMESTAMP_ORDER_DEFAULT
+ }
+ return p.INT96_TIMESTAMP_ORDER
+}
+
func (p *ColumnOrder) CountSetFieldsColumnOrder() int {
count := 0
if (p.IsSetTYPE_ORDER()) {
count++
}
+ if (p.IsSetIEEE_754_TOTAL_ORDER()) {
+ count++
+ }
+ if (p.IsSetINT96_TIMESTAMP_ORDER()) {
+ count++
+ }
return count
}
@@ -12556,6 +13048,14 @@ func (p *ColumnOrder) IsSetTYPE_ORDER() bool {
return p.TYPE_ORDER != nil
}
+func (p *ColumnOrder) IsSetIEEE_754_TOTAL_ORDER() bool {
+ return p.IEEE_754_TOTAL_ORDER != nil
+}
+
+func (p *ColumnOrder) IsSetINT96_TIMESTAMP_ORDER() bool {
+ return p.INT96_TIMESTAMP_ORDER != nil
+}
+
func (p *ColumnOrder) Read(ctx context.Context, iprot thrift.TProtocol) error {
if _, err := iprot.ReadStructBegin(ctx); err != nil {
return thrift.PrependError(fmt.Sprintf("%T read error: ", p),
err)
@@ -12581,6 +13081,26 @@ func (p *ColumnOrder) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return err
}
}
+ case 2:
+ if fieldTypeId == thrift.STRUCT {
+ if err := p.ReadField2(ctx, iprot); err != nil {
+ return err
+ }
+ } else {
+ if err := iprot.Skip(ctx, fieldTypeId); err !=
nil {
+ return err
+ }
+ }
+ case 3:
+ if fieldTypeId == thrift.STRUCT {
+ if err := p.ReadField3(ctx, iprot); err != nil {
+ return err
+ }
+ } else {
+ if err := iprot.Skip(ctx, fieldTypeId); err !=
nil {
+ return err
+ }
+ }
default:
if err := iprot.Skip(ctx, fieldTypeId); err != nil {
return err
@@ -12604,15 +13124,33 @@ func (p *ColumnOrder) ReadField1(ctx context.Context,
iprot thrift.TProtocol) er
return nil
}
+func (p *ColumnOrder) ReadField2(ctx context.Context, iprot thrift.TProtocol)
error {
+ p.IEEE_754_TOTAL_ORDER = &IEEE754TotalOrder{}
+ if err := p.IEEE_754_TOTAL_ORDER.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error reading
struct: ", p.IEEE_754_TOTAL_ORDER), err)
+ }
+ return nil
+}
+
+func (p *ColumnOrder) ReadField3(ctx context.Context, iprot thrift.TProtocol)
error {
+ p.INT96_TIMESTAMP_ORDER = &Int96TimestampOrder{}
+ if err := p.INT96_TIMESTAMP_ORDER.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error reading
struct: ", p.INT96_TIMESTAMP_ORDER), err)
+ }
+ return nil
+}
+
func (p *ColumnOrder) Write(ctx context.Context, oprot thrift.TProtocol) error
{
if c := p.CountSetFieldsColumnOrder(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "ColumnOrder"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
}
if p != nil {
if err := p.writeField1(ctx, oprot); err != nil { return err }
+ if err := p.writeField2(ctx, oprot); err != nil { return err }
+ if err := p.writeField3(ctx, oprot); err != nil { return err }
}
if err := oprot.WriteFieldStop(ctx); err != nil {
return thrift.PrependError("write field stop error: ", err)
@@ -12638,6 +13176,36 @@ func (p *ColumnOrder) writeField1(ctx context.Context,
oprot thrift.TProtocol) (
return err
}
+func (p *ColumnOrder) writeField2(ctx context.Context, oprot thrift.TProtocol)
(err error) {
+ if p.IsSetIEEE_754_TOTAL_ORDER() {
+ if err := oprot.WriteFieldBegin(ctx, "IEEE_754_TOTAL_ORDER",
thrift.STRUCT, 2); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
begin error 2:IEEE_754_TOTAL_ORDER: ", p), err)
+ }
+ if err := p.IEEE_754_TOTAL_ORDER.Write(ctx, oprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error
writing struct: ", p.IEEE_754_TOTAL_ORDER), err)
+ }
+ if err := oprot.WriteFieldEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
end error 2:IEEE_754_TOTAL_ORDER: ", p), err)
+ }
+ }
+ return err
+}
+
+func (p *ColumnOrder) writeField3(ctx context.Context, oprot thrift.TProtocol)
(err error) {
+ if p.IsSetINT96_TIMESTAMP_ORDER() {
+ if err := oprot.WriteFieldBegin(ctx, "INT96_TIMESTAMP_ORDER",
thrift.STRUCT, 3); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
begin error 3:INT96_TIMESTAMP_ORDER: ", p), err)
+ }
+ if err := p.INT96_TIMESTAMP_ORDER.Write(ctx, oprot); err != nil
{
+ return thrift.PrependError(fmt.Sprintf("%T error
writing struct: ", p.INT96_TIMESTAMP_ORDER), err)
+ }
+ if err := oprot.WriteFieldEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
end error 3:INT96_TIMESTAMP_ORDER: ", p), err)
+ }
+ }
+ return err
+}
+
func (p *ColumnOrder) Equals(other *ColumnOrder) bool {
if p == other {
return true
@@ -12645,6 +13213,8 @@ func (p *ColumnOrder) Equals(other *ColumnOrder) bool {
return false
}
if !p.TYPE_ORDER.Equals(other.TYPE_ORDER) { return false }
+ if !p.IEEE_754_TOTAL_ORDER.Equals(other.IEEE_754_TOTAL_ORDER) { return
false }
+ if !p.INT96_TIMESTAMP_ORDER.Equals(other.INT96_TIMESTAMP_ORDER) {
return false }
return true
}
@@ -12674,8 +13244,8 @@ func (p *ColumnOrder) Validate() error {
// Attributes:
// - Offset: Offset of the page in the file *
-// - CompressedPageSize: Size of the page, including header. Sum of
compressed_page_size and header
-// length
+// - CompressedPageSize: Size of the page, including header. Equal to the sum
of the page's
+// PageHeader.compressed_page_size and the size of the serialized PageHeader.
// - FirstRowIndex: Index within the RowGroup of the first row of the page.
When an
// OffsetIndex is present, pages must begin on row boundaries
// (repetition_level = 0).
@@ -12772,13 +13342,13 @@ func (p *PageLocation) Read(ctx context.Context,
iprot thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetOffset{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Offset is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Offset is not set"))
}
if !issetCompressedPageSize{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field CompressedPageSize is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field CompressedPageSize is not set"))
}
if !issetFirstRowIndex{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field FirstRowIndex is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field FirstRowIndex is not set"))
}
return nil
}
@@ -12914,7 +13484,7 @@ func (p *PageLocation) Validate() error {
// that page_locations[i].first_row_index <
page_locations[i+1].first_row_index.
// - UnencodedByteArrayDataBytes: Unencoded/uncompressed size for BYTE_ARRAY
types.
//
-// See documention for unencoded_byte_array_data_bytes in SizeStatistics for
+// See documentation for unencoded_byte_array_data_bytes in SizeStatistics for
// more details on this field.
//
type OffsetIndex struct {
@@ -12993,7 +13563,7 @@ func (p *OffsetIndex) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetPageLocations{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PageLocations is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field PageLocations is not set"))
}
return nil
}
@@ -13162,11 +13732,30 @@ func (p *OffsetIndex) Validate() error {
// - MinValues: Two lists containing lower and upper bounds for the values of
each page
// determined by the ColumnOrder of the column. These may be the actual
// minimum and maximum values found on a page, but can also be (more compact)
-// values that do not exist on a page. For example, instead of storing ""Blart
+// values that do not exist on a page. For example, instead of storing "Blart
// Versenwald III", a writer may set min_values[i]="B", max_values[i]="C".
// Such more compact values must still be valid values within the column's
// logical type. Readers must make sure that list entries are populated before
// using them by inspecting null_pages.
+//
+// For columns of physical type FLOAT or DOUBLE, or logical type FLOAT16,
+// NaN values are not to be included in these bounds. If all non-null values
+// of a page are NaN, then a writer must do the following:
+// - If the order of this column is TYPE_ORDER, then a column index must
+// not be written for this column chunk. While this is unfortunate for
+// performance, it is necessary to avoid conflict with legacy files that
+// still included NaN in min_values and max_values even if the page had
+// non-NaN values. To mitigate this, IEEE754_TOTAL_ORDER is recommended.
+// - If the order of this column is IEEE754_TOTAL_ORDER, then min_values[i]
+// and max_values[i] of that page must be set to the smallest and largest
+// NaN values as defined by IEEE 754 total order.
+//
+// For columns of physical type INT96, the writer must do the following:
+// - If the order of this column is not INT96_TIMESTAMP_ORDER, then a column
+// index must not be written for this column chunk.
+// - If the order of this column is INT96_TIMESTAMP_ORDER, the min_values[i]
+// and max_values[i] of that page must be set to the smallest and largest
+// values as defined by the INT96 chronological timestamp ordering.
// - MaxValues
// - BoundaryOrder: Stores whether both min_values and max_values are ordered
and if so, in
// which direction. This allows readers to perform binary searches in both
@@ -13193,6 +13782,10 @@ func (p *OffsetIndex) Validate() error {
//
// - DefinitionLevelHistograms: Same as repetition_level_histograms except
for definitions levels.
//
+// - NanCounts: A list containing the number of NaN values for each page.
Only present
+// for columns of physical type FLOAT or DOUBLE, or logical type FLOAT16.
+// If this field is not present, readers MUST assume that there might be
+// NaN values in any page.
//
type ColumnIndex struct {
NullPages []bool `thrift:"null_pages,1,required" db:"null_pages"
json:"null_pages"`
@@ -13202,6 +13795,7 @@ type ColumnIndex struct {
NullCounts []int64 `thrift:"null_counts,5" db:"null_counts"
json:"null_counts,omitempty"`
RepetitionLevelHistograms []int64
`thrift:"repetition_level_histograms,6" db:"repetition_level_histograms"
json:"repetition_level_histograms,omitempty"`
DefinitionLevelHistograms []int64
`thrift:"definition_level_histograms,7" db:"definition_level_histograms"
json:"definition_level_histograms,omitempty"`
+ NanCounts []int64 `thrift:"nan_counts,8" db:"nan_counts"
json:"nan_counts,omitempty"`
}
func NewColumnIndex() *ColumnIndex {
@@ -13253,6 +13847,13 @@ func (p *ColumnIndex) GetDefinitionLevelHistograms()
[]int64 {
return p.DefinitionLevelHistograms
}
+var ColumnIndex_NanCounts_DEFAULT []int64
+
+
+func (p *ColumnIndex) GetNanCounts() []int64 {
+ return p.NanCounts
+}
+
func (p *ColumnIndex) IsSetNullCounts() bool {
return p.NullCounts != nil
}
@@ -13265,6 +13866,10 @@ func (p *ColumnIndex) IsSetDefinitionLevelHistograms()
bool {
return p.DefinitionLevelHistograms != nil
}
+func (p *ColumnIndex) IsSetNanCounts() bool {
+ return p.NanCounts != nil
+}
+
func (p *ColumnIndex) Read(ctx context.Context, iprot thrift.TProtocol) error {
if _, err := iprot.ReadStructBegin(ctx); err != nil {
return thrift.PrependError(fmt.Sprintf("%T read error: ", p),
err)
@@ -13358,6 +13963,16 @@ func (p *ColumnIndex) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return err
}
}
+ case 8:
+ if fieldTypeId == thrift.LIST {
+ if err := p.ReadField8(ctx, iprot); err != nil {
+ return err
+ }
+ } else {
+ if err := iprot.Skip(ctx, fieldTypeId); err !=
nil {
+ return err
+ }
+ }
default:
if err := iprot.Skip(ctx, fieldTypeId); err != nil {
return err
@@ -13371,16 +13986,16 @@ func (p *ColumnIndex) Read(ctx context.Context, iprot
thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetNullPages{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NullPages is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NullPages is not set"))
}
if !issetMinValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field MinValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field MinValues is not set"))
}
if !issetMaxValues{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field MaxValues is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field MaxValues is not set"))
}
if !issetBoundaryOrder{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field BoundaryOrder is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field BoundaryOrder is not set"))
}
return nil
}
@@ -13527,6 +14142,28 @@ func (p *ColumnIndex) ReadField7(ctx context.Context,
iprot thrift.TProtocol) er
return nil
}
+func (p *ColumnIndex) ReadField8(ctx context.Context, iprot thrift.TProtocol)
error {
+ _, size, err := iprot.ReadListBegin(ctx)
+ if err != nil {
+ return thrift.PrependError("error reading list begin: ", err)
+ }
+ tSlice := make([]int64, 0, size)
+ p.NanCounts = tSlice
+ for i := 0; i < size; i++ {
+ var _elem30 int64
+ if v, err := iprot.ReadI64(ctx); err != nil {
+ return thrift.PrependError("error reading field 0: ",
err)
+ } else {
+ _elem30 = v
+ }
+ p.NanCounts = append(p.NanCounts, _elem30)
+ }
+ if err := iprot.ReadListEnd(ctx); err != nil {
+ return thrift.PrependError("error reading list end: ", err)
+ }
+ return nil
+}
+
func (p *ColumnIndex) Write(ctx context.Context, oprot thrift.TProtocol) error
{
if err := oprot.WriteStructBegin(ctx, "ColumnIndex"); err != nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -13539,6 +14176,7 @@ func (p *ColumnIndex) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if err := p.writeField5(ctx, oprot); err != nil { return err }
if err := p.writeField6(ctx, oprot); err != nil { return err }
if err := p.writeField7(ctx, oprot); err != nil { return err }
+ if err := p.writeField8(ctx, oprot); err != nil { return err }
}
if err := oprot.WriteFieldStop(ctx); err != nil {
return thrift.PrependError("write field stop error: ", err)
@@ -13694,6 +14332,29 @@ func (p *ColumnIndex) writeField7(ctx context.Context,
oprot thrift.TProtocol) (
return err
}
+func (p *ColumnIndex) writeField8(ctx context.Context, oprot thrift.TProtocol)
(err error) {
+ if p.IsSetNanCounts() {
+ if err := oprot.WriteFieldBegin(ctx, "nan_counts", thrift.LIST,
8); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
begin error 8:nan_counts: ", p), err)
+ }
+ if err := oprot.WriteListBegin(ctx, thrift.I64,
len(p.NanCounts)); err != nil {
+ return thrift.PrependError("error writing list begin:
", err)
+ }
+ for _, v := range p.NanCounts {
+ if err := oprot.WriteI64(ctx, int64(v)); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T. (0)
field write error: ", p), err)
+ }
+ }
+ if err := oprot.WriteListEnd(ctx); err != nil {
+ return thrift.PrependError("error writing list end: ",
err)
+ }
+ if err := oprot.WriteFieldEnd(ctx); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T write field
end error 8:nan_counts: ", p), err)
+ }
+ }
+ return err
+}
+
func (p *ColumnIndex) Equals(other *ColumnIndex) bool {
if p == other {
return true
@@ -13702,34 +14363,39 @@ func (p *ColumnIndex) Equals(other *ColumnIndex) bool
{
}
if len(p.NullPages) != len(other.NullPages) { return false }
for i, _tgt := range p.NullPages {
- _src30 := other.NullPages[i]
- if _tgt != _src30 { return false }
+ _src31 := other.NullPages[i]
+ if _tgt != _src31 { return false }
}
if len(p.MinValues) != len(other.MinValues) { return false }
for i, _tgt := range p.MinValues {
- _src31 := other.MinValues[i]
- if bytes.Compare(_tgt, _src31) != 0 { return false }
+ _src32 := other.MinValues[i]
+ if bytes.Compare(_tgt, _src32) != 0 { return false }
}
if len(p.MaxValues) != len(other.MaxValues) { return false }
for i, _tgt := range p.MaxValues {
- _src32 := other.MaxValues[i]
- if bytes.Compare(_tgt, _src32) != 0 { return false }
+ _src33 := other.MaxValues[i]
+ if bytes.Compare(_tgt, _src33) != 0 { return false }
}
if p.BoundaryOrder != other.BoundaryOrder { return false }
if len(p.NullCounts) != len(other.NullCounts) { return false }
for i, _tgt := range p.NullCounts {
- _src33 := other.NullCounts[i]
- if _tgt != _src33 { return false }
+ _src34 := other.NullCounts[i]
+ if _tgt != _src34 { return false }
}
if len(p.RepetitionLevelHistograms) !=
len(other.RepetitionLevelHistograms) { return false }
for i, _tgt := range p.RepetitionLevelHistograms {
- _src34 := other.RepetitionLevelHistograms[i]
- if _tgt != _src34 { return false }
+ _src35 := other.RepetitionLevelHistograms[i]
+ if _tgt != _src35 { return false }
}
if len(p.DefinitionLevelHistograms) !=
len(other.DefinitionLevelHistograms) { return false }
for i, _tgt := range p.DefinitionLevelHistograms {
- _src35 := other.DefinitionLevelHistograms[i]
- if _tgt != _src35 { return false }
+ _src36 := other.DefinitionLevelHistograms[i]
+ if _tgt != _src36 { return false }
+ }
+ if len(p.NanCounts) != len(other.NanCounts) { return false }
+ for i, _tgt := range p.NanCounts {
+ _src37 := other.NanCounts[i]
+ if _tgt != _src37 { return false }
}
return true
}
@@ -14361,7 +15027,7 @@ func (p *EncryptionAlgorithm) ReadField2(ctx
context.Context, iprot thrift.TProt
func (p *EncryptionAlgorithm) Write(ctx context.Context, oprot
thrift.TProtocol) error {
if c := p.CountSetFieldsEncryptionAlgorithm(); c != 1 {
- return fmt.Errorf("%T write union: exactly one field must be
set (%d set)", p, c)
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write
union: exactly one field must be set (%d set)", p, c))
}
if err := oprot.WriteStructBegin(ctx, "EncryptionAlgorithm"); err !=
nil {
return thrift.PrependError(fmt.Sprintf("%T write struct begin
error: ", p), err)
@@ -14447,7 +15113,13 @@ func (p *EncryptionAlgorithm) Validate() error {
// Description for file metadata
//
// Attributes:
-// - Version: Version of this file *
+// - Version: Version of this file
+//
+// As of December 2025, there is no agreed upon consensus of what constitutes
+// version 2 of the file. For maximum compatibility with readers, writers
should
+// always populate "1" for version. For maximum compatibility with writers,
+// readers should accept "1" and "2" interchangeably. All other versions are
+// reserved for potential future use-cases.
// - Schema: Parquet schema for this file. This schema contains metadata for
all the columns.
// The schema is represented as a tree with a single root. The nodes of the
tree
// are flattened to a list by doing a depth-first traversal.
@@ -14464,7 +15136,7 @@ func (p *EncryptionAlgorithm) Validate() error {
// - ColumnOrders: Sort order used for the min_value and max_value fields in
the Statistics
// objects and the min_values and max_values fields in the ColumnIndex
// objects of each column in this file. Sort orders are listed in the order
-// matching the columns in the schema. The indexes are not necessary the same
+// matching the columns in the schema. The indexes are not necessarily the same
// though, because only leaf nodes of the schema are represented in the list
// of sort orders.
//
@@ -14706,16 +15378,16 @@ func (p *FileMetaData) Read(ctx context.Context,
iprot thrift.TProtocol) error {
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetVersion{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Version is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Version is not set"))
}
if !issetSchema{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Schema is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field Schema is not set"))
}
if !issetNumRows{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field NumRows is not set"))
}
if !issetRowGroups{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RowGroups is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field RowGroups is not set"))
}
return nil
}
@@ -14737,11 +15409,11 @@ func (p *FileMetaData) ReadField2(ctx
context.Context, iprot thrift.TProtocol) e
tSlice := make([]*SchemaElement, 0, size)
p.Schema = tSlice
for i := 0; i < size; i++ {
- _elem36 := &SchemaElement{}
- if err := _elem36.Read(ctx, iprot); err != nil {
- return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem36), err)
+ _elem38 := &SchemaElement{}
+ if err := _elem38.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem38), err)
}
- p.Schema = append(p.Schema, _elem36)
+ p.Schema = append(p.Schema, _elem38)
}
if err := iprot.ReadListEnd(ctx); err != nil {
return thrift.PrependError("error reading list end: ", err)
@@ -14766,11 +15438,11 @@ func (p *FileMetaData) ReadField4(ctx
context.Context, iprot thrift.TProtocol) e
tSlice := make([]*RowGroup, 0, size)
p.RowGroups = tSlice
for i := 0; i < size; i++ {
- _elem37 := &RowGroup{}
- if err := _elem37.Read(ctx, iprot); err != nil {
- return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem37), err)
+ _elem39 := &RowGroup{}
+ if err := _elem39.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem39), err)
}
- p.RowGroups = append(p.RowGroups, _elem37)
+ p.RowGroups = append(p.RowGroups, _elem39)
}
if err := iprot.ReadListEnd(ctx); err != nil {
return thrift.PrependError("error reading list end: ", err)
@@ -14786,11 +15458,11 @@ func (p *FileMetaData) ReadField5(ctx
context.Context, iprot thrift.TProtocol) e
tSlice := make([]*KeyValue, 0, size)
p.KeyValueMetadata = tSlice
for i := 0; i < size; i++ {
- _elem38 := &KeyValue{}
- if err := _elem38.Read(ctx, iprot); err != nil {
- return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem38), err)
+ _elem40 := &KeyValue{}
+ if err := _elem40.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem40), err)
}
- p.KeyValueMetadata = append(p.KeyValueMetadata, _elem38)
+ p.KeyValueMetadata = append(p.KeyValueMetadata, _elem40)
}
if err := iprot.ReadListEnd(ctx); err != nil {
return thrift.PrependError("error reading list end: ", err)
@@ -14815,11 +15487,11 @@ func (p *FileMetaData) ReadField7(ctx
context.Context, iprot thrift.TProtocol) e
tSlice := make([]*ColumnOrder, 0, size)
p.ColumnOrders = tSlice
for i := 0; i < size; i++ {
- _elem39 := &ColumnOrder{}
- if err := _elem39.Read(ctx, iprot); err != nil {
- return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem39), err)
+ _elem41 := &ColumnOrder{}
+ if err := _elem41.Read(ctx, iprot); err != nil {
+ return thrift.PrependError(fmt.Sprintf("%T error
reading struct: ", _elem41), err)
}
- p.ColumnOrders = append(p.ColumnOrders, _elem39)
+ p.ColumnOrders = append(p.ColumnOrders, _elem41)
}
if err := iprot.ReadListEnd(ctx); err != nil {
return thrift.PrependError("error reading list end: ", err)
@@ -15036,19 +15708,19 @@ func (p *FileMetaData) Equals(other *FileMetaData)
bool {
if p.Version != other.Version { return false }
if len(p.Schema) != len(other.Schema) { return false }
for i, _tgt := range p.Schema {
- _src40 := other.Schema[i]
- if !_tgt.Equals(_src40) { return false }
+ _src42 := other.Schema[i]
+ if !_tgt.Equals(_src42) { return false }
}
if p.NumRows != other.NumRows { return false }
if len(p.RowGroups) != len(other.RowGroups) { return false }
for i, _tgt := range p.RowGroups {
- _src41 := other.RowGroups[i]
- if !_tgt.Equals(_src41) { return false }
+ _src43 := other.RowGroups[i]
+ if !_tgt.Equals(_src43) { return false }
}
if len(p.KeyValueMetadata) != len(other.KeyValueMetadata) { return
false }
for i, _tgt := range p.KeyValueMetadata {
- _src42 := other.KeyValueMetadata[i]
- if !_tgt.Equals(_src42) { return false }
+ _src44 := other.KeyValueMetadata[i]
+ if !_tgt.Equals(_src44) { return false }
}
if p.CreatedBy != other.CreatedBy {
if p.CreatedBy == nil || other.CreatedBy == nil {
@@ -15058,8 +15730,8 @@ func (p *FileMetaData) Equals(other *FileMetaData) bool
{
}
if len(p.ColumnOrders) != len(other.ColumnOrders) { return false }
for i, _tgt := range p.ColumnOrders {
- _src43 := other.ColumnOrders[i]
- if !_tgt.Equals(_src43) { return false }
+ _src45 := other.ColumnOrders[i]
+ if !_tgt.Equals(_src45) { return false }
}
if !p.EncryptionAlgorithm.Equals(other.EncryptionAlgorithm) { return
false }
if bytes.Compare(p.FooterSigningKeyMetadata,
other.FooterSigningKeyMetadata) != 0 { return false }
@@ -15182,7 +15854,7 @@ func (p *FileCryptoMetaData) Read(ctx context.Context,
iprot thrift.TProtocol) e
return thrift.PrependError(fmt.Sprintf("%T read struct end
error: ", p), err)
}
if !issetEncryptionAlgorithm{
- return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field EncryptionAlgorithm is not set"));
+ return
thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required
field EncryptionAlgorithm is not set"))
}
return nil
}
diff --git a/parquet/schema/logical_types.go b/parquet/schema/logical_types.go
index e7f1c29f..f1e85bd0 100644
--- a/parquet/schema/logical_types.go
+++ b/parquet/schema/logical_types.go
@@ -73,6 +73,10 @@ func getLogicalType(l *format.LogicalType) LogicalType {
return Float16LogicalType{}
case l.IsSetVARIANT():
return VariantLogicalType{}
+ case l.IsSetGEOGRAPHY():
+ return geographyLogicalTypeFromThrift(l.GEOGRAPHY)
+ case l.IsSetGEOMETRY():
+ return geometryLogicalTypeFromThrift(l.GEOMETRY)
case l == nil:
return NoLogicalType{}
default:
@@ -1228,3 +1232,257 @@ func (NoLogicalType) Equals(rhs LogicalType) bool {
}
func (NoLogicalType) IsNone() bool { return true }
+
+// GeometryLogicalType represents geospatial features in WKB format.
+type GeometryLogicalType struct {
+ baseLogicalType
+ // Crs optionally identifies the coordinate reference system. If empty,
+ // it defaults to OGC:CRS84.
+ Crs string
+}
+
+func geometryLogicalTypeFromThrift(t *format.GeometryType) GeometryLogicalType
{
+ if t == nil || !t.IsSetCrs() {
+ return GeometryLogicalType{}
+ }
+ return GeometryLogicalType{Crs: t.GetCrs()}
+}
+
+// CRS returns the coordinate reference system, or OGC:CRS84 when unset.
+func (t GeometryLogicalType) CRS() string {
+ if !t.IsCRSSet() {
+ return defaultGeospatialCRS
+ }
+ return t.Crs
+}
+
+// IsCRSSet returns whether the coordinate reference system was explicitly set.
+func (t GeometryLogicalType) IsCRSSet() bool {
+ return t.Crs != ""
+}
+
+func (t GeometryLogicalType) Equals(rhs LogicalType) bool {
+ other, ok := rhs.(GeometryLogicalType)
+ return ok && t.Crs == other.Crs
+}
+
+func (GeometryLogicalType) IsCompatible(c ConvertedType, dec DecimalMetadata)
bool {
+ if dec.IsSet {
+ return false
+ }
+ switch c {
+ case ConvertedTypes.None, ConvertedTypes.NA:
+ return true
+ }
+ return false
+}
+
+func (t GeometryLogicalType) MarshalJSON() ([]byte, error) {
+ values := map[string]interface{}{"Type": "Geometry"}
+ if t.IsCRSSet() {
+ values["crs"] = t.CRS()
+ }
+ return json.Marshal(values)
+}
+
+func (t GeometryLogicalType) String() string {
+ if t.IsCRSSet() {
+ return fmt.Sprintf("Geometry(crs=%s)", t.CRS())
+ }
+ return "Geometry"
+}
+
+func (GeometryLogicalType) SortOrder() SortOrder {
+ return SortUNKNOWN
+}
+
+func (GeometryLogicalType) ToConvertedType() (ConvertedType, DecimalMetadata) {
+ return ConvertedTypes.None, DecimalMetadata{}
+}
+
+func (GeometryLogicalType) IsApplicable(t parquet.Type, _ int32) bool {
+ return t == parquet.Types.ByteArray
+}
+
+func (t GeometryLogicalType) toThrift() *format.LogicalType {
+ typ := format.NewGeometryType()
+ if t.IsCRSSet() {
+ typ.Crs = thrift.StringPtr(t.Crs)
+ }
+ return &format.LogicalType{GEOMETRY: typ}
+}
+
+// GeographyLogicalType represents geospatial features in WKB format with a
+// non-linear edge interpolation algorithm.
+type GeographyLogicalType struct {
+ baseLogicalType
+ // Crs optionally identifies the coordinate reference system. If empty,
+ // it defaults to OGC:CRS84.
+ Crs string
+ // Algorithm optionally identifies the edge interpolation algorithm. If
+ // empty, it defaults to spherical.
+ Algorithm GeographyEdgeInterpolationAlgorithm
+}
+
+const defaultGeospatialCRS = "OGC:CRS84"
+
+func geographyLogicalTypeFromThrift(t *format.GeographyType)
GeographyLogicalType {
+ if t == nil {
+ return GeographyLogicalType{}
+ }
+
+ ret := GeographyLogicalType{}
+ if t.IsSetCrs() {
+ ret.Crs = t.GetCrs()
+ }
+ if t.IsSetAlgorithm() {
+ ret.Algorithm.fromThrift(t.GetAlgorithm())
+ }
+ return ret
+}
+
+// CRS returns the coordinate reference system, or OGC:CRS84 when unset.
+func (t GeographyLogicalType) CRS() string {
+ if !t.IsCRSSet() {
+ return defaultGeospatialCRS
+ }
+ return t.Crs
+}
+
+// IsCRSSet returns whether the coordinate reference system was explicitly set.
+func (t GeographyLogicalType) IsCRSSet() bool {
+ return t.Crs != ""
+}
+
+// EdgeInterpolationAlgorithm returns the edge interpolation algorithm. If
+// unset, it returns GeographyEdgeSpherical.
+func (t GeographyLogicalType) EdgeInterpolationAlgorithm()
GeographyEdgeInterpolationAlgorithm {
+ if !t.IsEdgeInterpolationAlgorithmSet() {
+ return GeographyEdgeSpherical
+ }
+ return t.Algorithm
+}
+
+// IsEdgeInterpolationAlgorithmSet returns whether the edge interpolation
+// algorithm was explicitly set.
+func (t GeographyLogicalType) IsEdgeInterpolationAlgorithmSet() bool {
+ return t.Algorithm != ""
+}
+
+func (t GeographyLogicalType) Equals(rhs LogicalType) bool {
+ other, ok := rhs.(GeographyLogicalType)
+ return ok && t.Crs == other.Crs && t.Algorithm == other.Algorithm
+}
+
+func (GeographyLogicalType) IsCompatible(c ConvertedType, dec DecimalMetadata)
bool {
+ if dec.IsSet {
+ return false
+ }
+ switch c {
+ case ConvertedTypes.None, ConvertedTypes.NA:
+ return true
+ }
+ return false
+}
+
+func (t GeographyLogicalType) MarshalJSON() ([]byte, error) {
+ values := map[string]any{"Type": "Geography"}
+ if t.IsCRSSet() {
+ values["crs"] = t.CRS()
+ }
+ if t.IsEdgeInterpolationAlgorithmSet() {
+ values["algorithm"] = t.EdgeInterpolationAlgorithm().String()
+ }
+ return json.Marshal(values)
+}
+
+func (t GeographyLogicalType) String() string {
+ if !t.IsCRSSet() && !t.IsEdgeInterpolationAlgorithmSet() {
+ return "Geography"
+ }
+ return fmt.Sprintf("Geography(crs=%s, algorithm=%s)", t.CRS(),
t.EdgeInterpolationAlgorithm())
+}
+
+func (GeographyLogicalType) SortOrder() SortOrder {
+ return SortUNKNOWN
+}
+
+func (GeographyLogicalType) ToConvertedType() (ConvertedType, DecimalMetadata)
{
+ return ConvertedTypes.None, DecimalMetadata{}
+}
+
+func (GeographyLogicalType) IsApplicable(t parquet.Type, _ int32) bool {
+ return t == parquet.Types.ByteArray
+}
+
+func (t GeographyLogicalType) toThrift() *format.LogicalType {
+ typ := format.NewGeographyType()
+ if t.IsCRSSet() {
+ typ.Crs = thrift.StringPtr(t.Crs)
+ }
+ if t.IsEdgeInterpolationAlgorithmSet() {
+ alg := t.Algorithm.toThrift()
+ typ.Algorithm = &alg
+ }
+ return &format.LogicalType{GEOGRAPHY: typ}
+}
+
+// GeographyEdgeInterpolationAlgorithm describes how geography edges are
+// interpolated between points.
+type GeographyEdgeInterpolationAlgorithm string
+
+// Constants for the GeographyEdgeInterpolationAlgorithm.
+const (
+ GeographyEdgeSpherical GeographyEdgeInterpolationAlgorithm = "spherical"
+ GeographyEdgeVincenty GeographyEdgeInterpolationAlgorithm = "vincenty"
+ GeographyEdgeThomas GeographyEdgeInterpolationAlgorithm = "thomas"
+ GeographyEdgeAndoyer GeographyEdgeInterpolationAlgorithm = "andoyer"
+ GeographyEdgeKarney GeographyEdgeInterpolationAlgorithm = "karney"
+)
+
+func (alg *GeographyEdgeInterpolationAlgorithm) fromThrift(thriftAlg
format.EdgeInterpolationAlgorithm) {
+ switch thriftAlg {
+ case format.EdgeInterpolationAlgorithm_VINCENTY:
+ *alg = GeographyEdgeVincenty
+ case format.EdgeInterpolationAlgorithm_THOMAS:
+ *alg = GeographyEdgeThomas
+ case format.EdgeInterpolationAlgorithm_ANDOYER:
+ *alg = GeographyEdgeAndoyer
+ case format.EdgeInterpolationAlgorithm_KARNEY:
+ *alg = GeographyEdgeKarney
+ case format.EdgeInterpolationAlgorithm_SPHERICAL:
+ fallthrough
+ default:
+ *alg = GeographyEdgeSpherical
+ }
+}
+
+func (alg GeographyEdgeInterpolationAlgorithm) toThrift()
format.EdgeInterpolationAlgorithm {
+ switch alg {
+ case GeographyEdgeVincenty:
+ return format.EdgeInterpolationAlgorithm_VINCENTY
+ case GeographyEdgeThomas:
+ return format.EdgeInterpolationAlgorithm_THOMAS
+ case GeographyEdgeAndoyer:
+ return format.EdgeInterpolationAlgorithm_ANDOYER
+ case GeographyEdgeKarney:
+ return format.EdgeInterpolationAlgorithm_KARNEY
+ case GeographyEdgeSpherical:
+ fallthrough
+ default:
+ return format.EdgeInterpolationAlgorithm_SPHERICAL
+ }
+}
+
+func (alg GeographyEdgeInterpolationAlgorithm) String() string {
+ switch alg {
+ case GeographyEdgeVincenty,
+ GeographyEdgeThomas,
+ GeographyEdgeAndoyer,
+ GeographyEdgeKarney,
+ GeographyEdgeSpherical:
+ return string(alg)
+ default:
+ return string(GeographyEdgeSpherical)
+ }
+}
diff --git a/parquet/schema/logical_types_test.go
b/parquet/schema/logical_types_test.go
index 62d9294b..f3e21333 100644
--- a/parquet/schema/logical_types_test.go
+++ b/parquet/schema/logical_types_test.go
@@ -228,6 +228,8 @@ func TestLogicalTypeProperties(t *testing.T) {
{"uuid", schema.UUIDLogicalType{}, false, true, true},
{"float16", schema.Float16LogicalType{}, false, true, true},
{"variant", schema.VariantLogicalType{}, true, true, true},
+ {"geometry", schema.GeometryLogicalType{}, false, true, true},
+ {"geography", schema.GeographyLogicalType{}, false, true, true},
{"nological", schema.NoLogicalType{}, false, false, true},
{"unknown", schema.UnknownLogicalType{}, false, false, false},
}
@@ -277,6 +279,8 @@ func TestLogicalSingleTypeApplicability(t *testing.T) {
{"int64", schema.NewIntLogicalType(64 /* bitWidth */, true /*
signed */), parquet.Types.Int64},
{"json", schema.JSONLogicalType{}, parquet.Types.ByteArray},
{"bson", schema.BSONLogicalType{}, parquet.Types.ByteArray},
+ {"geometry", schema.GeometryLogicalType{},
parquet.Types.ByteArray},
+ {"geography", schema.GeographyLogicalType{},
parquet.Types.ByteArray},
}
for _, tt := range tests {
@@ -459,6 +463,10 @@ func TestLogicalTypeRepresentation(t *testing.T) {
{"uuid", schema.UUIDLogicalType{}, "UUID", `{"Type": "UUID"}`},
{"float16", schema.Float16LogicalType{}, "Float16", `{"Type":
"Float16"}`},
{"variant", schema.VariantLogicalType{}, "Variant", `{"Type":
"Variant"}`},
+ {"geometry", schema.GeometryLogicalType{}, "Geometry",
`{"Type": "Geometry"}`},
+ {"geometry crs", schema.GeometryLogicalType{Crs: "EPSG:4326"},
"Geometry(crs=EPSG:4326)", `{"Type": "Geometry", "crs": "EPSG:4326"}`},
+ {"geography", schema.GeographyLogicalType{}, "Geography",
`{"Type": "Geography"}`},
+ {"geography crs algorithm", schema.GeographyLogicalType{Crs:
"OGC:CRS84", Algorithm: schema.GeographyEdgeKarney}, "Geography(crs=OGC:CRS84,
algorithm=karney)", `{"Type": "Geography", "crs": "OGC:CRS84", "algorithm":
"karney"}`},
{"none", schema.NoLogicalType{}, "None", `{"Type": "None"}`},
}
@@ -472,6 +480,18 @@ func TestLogicalTypeRepresentation(t *testing.T) {
}
}
+func TestGeospatialLogicalTypeDefaults(t *testing.T) {
+ geometry := schema.GeometryLogicalType{}
+ assert.False(t, geometry.IsCRSSet())
+ assert.Equal(t, "OGC:CRS84", geometry.CRS())
+
+ geography := schema.GeographyLogicalType{}
+ assert.False(t, geography.IsCRSSet())
+ assert.Equal(t, "OGC:CRS84", geography.CRS())
+ assert.False(t, geography.IsEdgeInterpolationAlgorithmSet())
+ assert.Equal(t, schema.GeographyEdgeSpherical,
geography.EdgeInterpolationAlgorithm())
+}
+
func TestLogicalTypeSortOrder(t *testing.T) {
tests := []struct {
name string
@@ -506,6 +526,8 @@ func TestLogicalTypeSortOrder(t *testing.T) {
{"uuid", schema.UUIDLogicalType{}, schema.SortUNSIGNED},
{"float16", schema.Float16LogicalType{}, schema.SortSIGNED},
{"variant", schema.VariantLogicalType{}, schema.SortUNKNOWN},
+ {"geometry", schema.GeometryLogicalType{}, schema.SortUNKNOWN},
+ {"geography", schema.GeographyLogicalType{},
schema.SortUNKNOWN},
{"none", schema.NoLogicalType{}, schema.SortUNKNOWN},
}
diff --git a/parquet/schema/schema_element_test.go
b/parquet/schema/schema_element_test.go
index a6d1538e..4bb32c7a 100644
--- a/parquet/schema/schema_element_test.go
+++ b/parquet/schema/schema_element_test.go
@@ -163,6 +163,14 @@ func (s *SchemaElementConstructionSuite) TestSimple() {
"float16", Float16LogicalType{},
parquet.Types.FixedLenByteArray, 2, false, ConvertedTypes.NA, true,
func(e *format.SchemaElement) bool { return
e.LogicalType.IsSetFLOAT16() },
}, nil},
+ {"geometry", &schemaElementConstructArgs{
+ "geometry", GeometryLogicalType{},
parquet.Types.ByteArray, -1, false, ConvertedTypes.NA, true,
+ func(e *format.SchemaElement) bool { return
e.LogicalType.IsSetGEOMETRY() },
+ }, nil},
+ {"geography", &schemaElementConstructArgs{
+ "geography", GeographyLogicalType{},
parquet.Types.ByteArray, -1, false, ConvertedTypes.NA, true,
+ func(e *format.SchemaElement) bool { return
e.LogicalType.IsSetGEOGRAPHY() },
+ }, nil},
{"none", &schemaElementConstructArgs{
"none", NoLogicalType{}, parquet.Types.Int64, -1,
false, ConvertedTypes.NA, false,
checkNone,
@@ -493,6 +501,8 @@ func TestLogicalTypeSerializationRoundTrip(t *testing.T) {
{"bson", BSONLogicalType{}, parquet.Types.ByteArray, -1},
{"uuid", UUIDLogicalType{}, parquet.Types.FixedLenByteArray,
16},
{"float16", Float16LogicalType{},
parquet.Types.FixedLenByteArray, 2},
+ {"geometry", GeometryLogicalType{Crs: "EPSG:4326"},
parquet.Types.ByteArray, -1},
+ {"geography", GeographyLogicalType{Crs: "OGC:CRS84", Algorithm:
GeographyEdgeKarney}, parquet.Types.ByteArray, -1},
{"none", NoLogicalType{}, parquet.Types.Boolean, -1},
}
diff --git a/parquet/variant/basic_type_stringer.go
b/parquet/variant/basic_type_stringer.go
index e8cf83e4..bf168b51 100644
--- a/parquet/variant/basic_type_stringer.go
+++ b/parquet/variant/basic_type_stringer.go
@@ -20,9 +20,9 @@ const _BasicType_name =
"UnknownPrimitiveShortStringObjectArray"
var _BasicType_index = [...]uint8{0, 7, 16, 27, 33, 38}
func (i BasicType) String() string {
- i -= -1
- if i < 0 || i >= BasicType(len(_BasicType_index)-1) {
- return "BasicType(" + strconv.FormatInt(int64(i+-1), 10) + ")"
+ idx := int(i) - -1
+ if i < -1 || idx >= len(_BasicType_index)-1 {
+ return "BasicType(" + strconv.FormatInt(int64(i), 10) + ")"
}
- return _BasicType_name[_BasicType_index[i]:_BasicType_index[i+1]]
+ return _BasicType_name[_BasicType_index[idx]:_BasicType_index[idx+1]]
}
diff --git a/parquet/variant/primitive_type_stringer.go
b/parquet/variant/primitive_type_stringer.go
index 205724c3..89f94cb8 100644
--- a/parquet/variant/primitive_type_stringer.go
+++ b/parquet/variant/primitive_type_stringer.go
@@ -37,9 +37,9 @@ const _PrimitiveType_name =
"UnknownNullBoolTrueBoolFalseInt8Int16Int32Int64Doub
var _PrimitiveType_index = [...]uint8{0, 7, 11, 19, 28, 32, 37, 42, 47, 53,
62, 71, 81, 85, 102, 122, 127, 133, 139, 154, 170, 189, 193}
func (i PrimitiveType) String() string {
- i -= -1
- if i < 0 || i >= PrimitiveType(len(_PrimitiveType_index)-1) {
- return "PrimitiveType(" + strconv.FormatInt(int64(i+-1), 10) +
")"
+ idx := int(i) - -1
+ if i < -1 || idx >= len(_PrimitiveType_index)-1 {
+ return "PrimitiveType(" + strconv.FormatInt(int64(i), 10) + ")"
}
- return
_PrimitiveType_name[_PrimitiveType_index[i]:_PrimitiveType_index[i+1]]
+ return
_PrimitiveType_name[_PrimitiveType_index[idx]:_PrimitiveType_index[idx+1]]
}
diff --git a/parquet/version_string.go b/parquet/version_string.go
index ab01aa48..c185d9b7 100644
--- a/parquet/version_string.go
+++ b/parquet/version_string.go
@@ -18,8 +18,9 @@ const _Version_name = "v1.0v2.4v2.6"
var _Version_index = [...]uint8{0, 4, 8, 12}
func (i Version) String() string {
- if i < 0 || i >= Version(len(_Version_index)-1) {
+ idx := int(i) - 0
+ if i < 0 || idx >= len(_Version_index)-1 {
return "Version(" + strconv.FormatInt(int64(i), 10) + ")"
}
- return _Version_name[_Version_index[i]:_Version_index[i+1]]
+ return _Version_name[_Version_index[idx]:_Version_index[idx+1]]
}
