ggershinsky commented on code in PR #7387: URL: https://github.com/apache/arrow-rs/pull/7387#discussion_r2034516853
########## parquet/src/encryption/key_management/mod.rs: ########## @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encryption Key Management Tools for Parquet +//! +//! This module provides tools for integrating with a Key Management Server (KMS) +//! to read and write encrypted Parquet files. +//! +//! Envelope encryption is used, where the Parquet file is encrypted with data encryption keys +//! (DEKs) that are randomly generated per file, +//! and the DEKs are encrypted with master encryption keys (MEKs) that are managed by a KMS. +//! Double wrapping is used by default, where the DEKs are first encrypted with key encryption +//! keys (KEKs) that are then encrypted with MEKs, to reduce KMS interactions. +//! +//! Using this module requires defining your own type that implements the +//! [`KmsClient`](kms::KmsClient) trait and interacts with your organization's KMS. +//! +//! This `KmsClient` can then be used by the +//! [`CryptoFactory`](crypto_factory::CryptoFactory) type to generate +//! [`FileEncryptionProperties`](crate::encryption::encrypt::FileEncryptionProperties) +//! for writing encrypted Parquet files and +//! [`FileDecryptionProperties`](crate::encryption::decrypt::FileDecryptionProperties) +//! for reading files. +//! +//! The encryption key metadata that is stored in the Parquet file is compatible with other Parquet +//! implementations (PyArrow and parquet-java for example), so that files encrypted with this +//! module may be decrypted by those implementations, and vice versa, as long as the +//! `KmsClient` implementations are compatible. +//! +//! # Example of writing then reading an encrypted Parquet file +//! ``` +//! use arrow_array::{ArrayRef, Float32Array, Int32Array, RecordBatch}; +//! use base64::prelude::BASE64_STANDARD; +//! use base64::Engine; +//! use parquet::arrow::arrow_reader::{ArrowReaderOptions, ParquetRecordBatchReaderBuilder}; +//! use parquet::arrow::ArrowWriter; +//! use parquet::encryption::key_management::crypto_factory::{ +//! CryptoFactory, DecryptionConfiguration, EncryptionConfigurationBuilder, +//! }; +//! use parquet::encryption::key_management::kms::{KmsClient, KmsConnectionConfig}; +//! use parquet::errors::{ParquetError, Result}; +//! use parquet::file::properties::WriterProperties; +//! use ring::aead::{Aad, LessSafeKey, UnboundKey, AES_128_GCM, NONCE_LEN}; +//! use ring::rand::{SecureRandom, SystemRandom}; +//! use std::collections::HashMap; +//! use std::fs::File; +//! use std::sync::Arc; +//! use tempfile::TempDir; +//! +//! let temp_dir = TempDir::new()?; +//! let file_path = temp_dir.path().join("encrypted_example.parquet"); +//! +//! // Create a CryptoFactory, providing a factory function +//! // that will create an example KMS client +//! let crypto_factory = CryptoFactory::new(DemoKmsClient::create); +//! +//! // Specify any options required to connect to our KMS. +//! // These are ignored by the DemoKmsClient but shown here for illustration. +//! // The KMS instance ID and URL will be stored in the Parquet encryption metadata +//! // so don't need to be specified if you are only reading files. +//! let connection_config = Arc::new( +//! KmsConnectionConfig::builder() +//! .set_kms_instance_id("kms1".into()) +//! .set_kms_instance_url("https://example.com/kms".into()) +//! .set_key_access_token("secret_token".into()) +//! .set_custom_kms_conf_option("custom_option".into(), "some_value".into()) +//! .build(), +//! ); +//! +//! // Create an encryption configuration that will encrypt the footer with the "kf" key, +//! // the "x" column with the "kc1" key, and the "y" column with the "kc2" key, +//! // while leaving the "id" column unencrypted. +//! let encryption_config = EncryptionConfigurationBuilder::new("kf".into()) +//! .add_column_key("kc1".into(), vec!["x".into()]) +//! .add_column_key("kc2".into(), vec!["x".into()]) Review Comment: should probably be the y column -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
