This is an automated email from the ASF dual-hosted git repository.
zeroshade pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-go.git
The following commit(s) were added to refs/heads/main by this push:
new d12c4280 fix(parquet): preserve CTR algorithm in plaintext footers
(#1412)
d12c4280 is described below
commit d12c4280ff365ca71efde1029084724dc1be5b61
Author: autoCAT <[email protected]>
AuthorDate: Tue Oct 27 08:08:34 2026 -0700
fix(parquet): preserve CTR algorithm in plaintext footers (#1412)
### Rationale for this change
Writing an AES_GCM_CTR_V1 file with a plaintext footer currently records
AES_GCM_V1 in the footer metadata. Readers then select GCM for the CTR
data pages and fail to read them. This also reproduces with arrow-go
v18.8.0.
Related to #566 and apache/arrow#54884.
### What changes are included in this PR?
Serialize the configured file algorithm directly, preserving its AAD
settings. Footer signing is unchanged.
Add a generated-file regression for both algorithms, encrypted and
plaintext footers, and stored or externally supplied AAD prefixes. It
checks serialized metadata and round-trip values, plus rejection of a
corrupted plaintext-footer signature.
### Are these changes tested?
The regression fails on the original code in all three CTR/plaintext
cases and passes with the fix.
* All 18 Parquet packages pass with `-race -tags assert`. The focused
regression also passes with `assert,noasm`.
* Files from this commit read correctly with arrow-go v18.8.0, the C++
reader from apache/arrow#54884, and parquet-java 1.18.1, including all
column values.
* Vet and lint pass for the changed packages. Apache RAT passes. Broader
vet/lint checks report only the existing BitRun benchmark warning and
three schema QF1001 suggestions, also reproduced on the unchanged base.
### Are there any user-facing changes?
New CTR files with plaintext footers declare the correct cipher and can
be read back. No API changes.
AI was used for the implementation, tests and PR description.
---
parquet/file/file_reader_test.go | 100 +++++++++++++++++++++++++++++++++++++++
parquet/metadata/file.go | 10 +---
2 files changed, 101 insertions(+), 9 deletions(-)
diff --git a/parquet/file/file_reader_test.go b/parquet/file/file_reader_test.go
index 9fa38c3c..2dd63e39 100644
--- a/parquet/file/file_reader_test.go
+++ b/parquet/file/file_reader_test.go
@@ -1043,3 +1043,103 @@ func TestListColumns(t *testing.T) {
}
}
}
+
+func TestEncryptedFileCipherAndAAD(t *testing.T) {
+ const footerKey = "0123456789012345"
+ values := []int32{3, 7, 11}
+ sc, err := schema.NewGroupNode("schema", parquet.Repetitions.Required,
+ schema.FieldList{schema.NewInt32Node("value",
parquet.Repetitions.Required, -1)}, -1)
+ require.NoError(t, err)
+ for _, cipher := range []struct {
+ name string
+ algorithm parquet.Cipher
+ }{
+ {"GCM", parquet.AesGcm}, {"CTR", parquet.AesCtr},
+ } {
+ for _, footer := range []struct {
+ name string
+ plaintext bool
+ aad string
+ externalAAD bool
+ }{
+ {"encrypted", false, "", false},
+ {"plaintext", true, "", false},
+ {"plaintext_stored_aad", true, "test-file", false},
+ {"plaintext_external_aad", true, "test-file", true},
+ } {
+ t.Run(fmt.Sprintf("%s/%s", cipher.name, footer.name),
func(t *testing.T) {
+ opts :=
[]parquet.EncryptOption{parquet.WithAlg(cipher.algorithm),
parquet.WithFooterKeyMetadata("footer-key")}
+ if footer.plaintext {
+ opts = append(opts,
parquet.WithPlaintextFooter())
+ }
+ if footer.aad != "" {
+ opts = append(opts,
parquet.WithAadPrefix(footer.aad))
+ }
+ if footer.externalAAD {
+ opts = append(opts,
parquet.DisableAadPrefixStorage())
+ }
+ encryption :=
parquet.NewFileEncryptionProperties(footerKey, opts...)
+ props :=
parquet.NewWriterProperties(parquet.WithEncryptionProperties(encryption),
parquet.WithDictionaryDefault(false))
+ var buf bytes.Buffer
+ writer, err :=
file.NewParquetWriterWithError(&buf, sc, file.WithWriterProps(props))
+ require.NoError(t, err)
+ group, err := writer.AppendRowGroupChecked()
+ require.NoError(t, err)
+ column, err := group.NextColumn()
+ require.NoError(t, err)
+ n, err :=
column.(*file.Int32ColumnChunkWriter).WriteBatch(values, nil, nil)
+ require.NoError(t, err)
+ require.EqualValues(t, len(values), n)
+ require.NoError(t, group.Close())
+ require.NoError(t, writer.Close())
+
+ if footer.plaintext {
+ // Inspect the serialized footer,
including the AAD storage policy.
+ data := buf.Bytes()
+ footerLength :=
int(binary.LittleEndian.Uint32(data[len(data)-8:]))
+ meta, err :=
metadata.NewFileMetaData(data[len(data)-8-footerLength:len(data)-8], nil)
+ require.NoError(t, err)
+ assert.Equal(t, encryption.Algorithm(),
meta.EncryptionAlgorithm())
+ assert.Equal(t, []byte("footer-key"),
meta.GetFooterSigningKeyMetadata())
+ }
+
+ alloc :=
memory.NewCheckedAllocator(memory.NewGoAllocator())
+ defer alloc.AssertSize(t, 0)
+
+ readerProps := func() *parquet.ReaderProperties
{
+ decryption :=
[]parquet.FileDecryptionOption{parquet.WithFooterKey(footerKey)}
+ if footer.externalAAD {
+ decryption = append(decryption,
parquet.WithDecryptAadPrefix(footer.aad))
+ }
+ p := parquet.NewReaderProperties(alloc)
+ p.FileDecryptProps =
parquet.NewFileDecryptionProperties(decryption...)
+ return p
+ }
+ reader, err :=
file.NewParquetReader(bytes.NewReader(buf.Bytes()),
file.WithReadProps(readerProps()))
+ require.NoError(t, err)
+ defer reader.Close()
+ require.EqualValues(t, len(values),
reader.NumRows())
+ readColumn, err := reader.RowGroup(0).Column(0)
+ require.NoError(t, err)
+ defer readColumn.Close()
+ actual := make([]int32, len(values))
+ total, count, err :=
readColumn.(*file.Int32ColumnChunkReader).ReadBatch(int64(len(values)), actual,
nil, nil)
+ require.NoError(t, err)
+ require.NoError(t, readColumn.Err())
+ require.EqualValues(t, len(values), total)
+ require.Equal(t, len(values), count)
+ assert.Equal(t, values, actual)
+ assert.False(t, readColumn.HasNext())
+ require.NoError(t, readColumn.Err())
+
+ if cipher.algorithm == parquet.AesCtr &&
footer.plaintext && footer.aad == "" {
+ // CTR data pages still use a GCM
signature for a plaintext footer.
+ corrupted := bytes.Clone(buf.Bytes())
+ corrupted[len(corrupted)-9] ^= 1
+ _, err :=
file.NewParquetReader(bytes.NewReader(corrupted),
file.WithReadProps(readerProps()))
+ require.ErrorContains(t, err,
"signature verification failed")
+ }
+ })
+ }
+ }
+}
diff --git a/parquet/metadata/file.go b/parquet/metadata/file.go
index 20a1a2ab..b10ae9ce 100644
--- a/parquet/metadata/file.go
+++ b/parquet/metadata/file.go
@@ -153,15 +153,7 @@ func (f *FileMetaDataBuilder) Snapshot() (*FileMetaData,
error) {
encryptProps := f.props.FileEncryptionProperties()
if encryptProps != nil && !encryptProps.EncryptedFooter() {
- var signingAlgo parquet.Algorithm
- algo := encryptProps.Algorithm()
- signingAlgo.Aad.AadFileUnique = algo.Aad.AadFileUnique
- signingAlgo.Aad.SupplyAadPrefix = algo.Aad.SupplyAadPrefix
- if !algo.Aad.SupplyAadPrefix {
- signingAlgo.Aad.AadPrefix = algo.Aad.AadPrefix
- }
- signingAlgo.Algo = parquet.AesGcm
- f.metadata.EncryptionAlgorithm = signingAlgo.ToThrift()
+ f.metadata.EncryptionAlgorithm =
encryptProps.Algorithm().ToThrift()
footerSigningMetadata :=
f.props.FileEncryptionProperties().FooterKeyMetadata()
if footerSigningMetadata != "" {
f.metadata.FooterSigningKeyMetadata =
[]byte(footerSigningMetadata)
