wgtmac commented on code in PR #1071: URL: https://github.com/apache/iceberg-cpp/pull/1071#discussion_r4611952167
########## src/iceberg/encryption/standard_encryption_manager.cc: ########## @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/encryption/standard_encryption_manager.h" + +#include <array> +#include <charconv> +#include <optional> + +#include "iceberg/encryption/aes_gcm_internal.h" +#include "iceberg/encryption/aes_gcm_stream.h" +#include "iceberg/util/base64.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +namespace { + +using encryption::AesGcmCipher; +using encryption::SecureRandomBytes; + +std::span<const uint8_t> AsBytes(std::string_view s) { + return {reinterpret_cast<const uint8_t*>(s.data()), s.size()}; +} + +std::string ToString(std::span<const uint8_t> bytes) { + return {reinterpret_cast<const char*>(bytes.data()), bytes.size()}; +} + +Result<std::string> GenerateKeyId() { + std::array<uint8_t, 16> id{}; + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(id)); + return Base64::Encode(ToString(id)); +} + +std::optional<int64_t> ParseTimestamp(const EncryptedKey& key) { + auto it = key.properties.find(std::string(StandardEncryptionManager::kKeyTimestamp)); + if (it == key.properties.end()) { + return std::nullopt; + } + int64_t value = 0; + const auto& str = it->second; + auto [ptr, ec] = std::from_chars(str.data(), str.data() + str.size(), value); + if (ec != std::errc{} || ptr != str.data() + str.size() || value < 0) { + return std::nullopt; + } + return value; +} + +} // namespace + +Result<std::shared_ptr<StandardEncryptionManager>> StandardEncryptionManager::Make( + EncryptionManagerRegistry::Options options) { + ICEBERG_PRECHECK(!options.table_key_id.empty(), "Invalid encryption key ID: empty"); + ICEBERG_PRECHECK(options.data_key_length == 16 || options.data_key_length == 24 || + options.data_key_length == 32, + "Invalid data key length: {} (must be 16, 24, or 32)", + options.data_key_length); + ICEBERG_PRECHECK(options.kms != nullptr, "Invalid KMS client: null"); + return std::shared_ptr<StandardEncryptionManager>( + new StandardEncryptionManager(std::move(options))); +} + +StandardEncryptionManager::StandardEncryptionManager( + EncryptionManagerRegistry::Options options) + : table_key_id_(std::move(options.table_key_id)), + data_key_length_(options.data_key_length), + kms_(std::move(options.kms)) { + for (auto& key : options.encryption_keys) { + AddKey(std::move(key)); + } +} + +void StandardEncryptionManager::AddKey(EncryptedKey key) { + auto key_id = key.key_id; + if (keys_.insert_or_assign(key_id, std::move(key)).second) { + key_order_.push_back(std::move(key_id)); + } +} + +void StandardEncryptionManager::SetTestTimeShift(std::chrono::milliseconds shift) { + std::lock_guard lock(mutex_); + time_shift_ = shift; +} + +int64_t StandardEncryptionManager::NowMs() const { + return std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::system_clock::now().time_since_epoch() + time_shift_) + .count(); +} + +Result<StandardKeyMetadata> StandardEncryptionManager::NewKeyMetadata() { + StandardKeyMetadata metadata; + metadata.encryption_key.resize(data_key_length_); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(metadata.encryption_key)); + metadata.aad_prefix.emplace(kAadPrefixLength); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(*metadata.aad_prefix)); + return metadata; +} + +Result<std::unique_ptr<InputFile>> StandardEncryptionManager::NewDecryptingInputFile( + std::unique_ptr<InputFile> encrypted, std::span<const uint8_t> key_metadata) { + ICEBERG_PRECHECK(encrypted != nullptr, "Encrypted input file cannot be null"); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, StandardKeyMetadata::Parse(key_metadata)); + if (!metadata.file_length.has_value()) { + // The length must come from trusted key metadata, not from the file system, so that + // truncation is detected (format/gcm-stream-spec.md, "File length"). + return Invalid("Cannot read AES GCM stream {}: key metadata has no file length", + encrypted->location()); + } + return std::make_unique<AesGcmInputFile>( + std::move(encrypted), std::move(metadata.encryption_key), + metadata.aad_prefix.value_or(std::vector<uint8_t>{}), *metadata.file_length); +} + +Result<std::unique_ptr<OutputFile>> StandardEncryptionManager::NewEncryptingOutputFile( + std::unique_ptr<OutputFile> plain, const StandardKeyMetadata& key_metadata) { + ICEBERG_PRECHECK(plain != nullptr, "Output file cannot be null"); + return std::make_unique<AesGcmOutputFile>( + std::move(plain), key_metadata.encryption_key, + key_metadata.aad_prefix.value_or(std::vector<uint8_t>{})); +} + +Result<std::vector<uint8_t>> StandardEncryptionManager::UnwrapKek( + const std::string& kek_id) { + int64_t now = NowMs(); + if (auto it = unwrapped_keks_.find(kek_id); it != unwrapped_keks_.end()) { + if (now - it->second.unwrapped_at_ms < kUnwrappedKekCacheExpiry.count()) { + return it->second.key; + } + unwrapped_keks_.erase(it); + } + auto it = keys_.find(kek_id); + if (it == keys_.end()) { + return Invalid("Cannot find key encryption key with id {}", kek_id); + } + ICEBERG_ASSIGN_OR_RAISE( + auto kek, + kms_->UnwrapKey(AsBytes(it->second.encrypted_key_metadata), table_key_id_)); + unwrapped_keks_.insert_or_assign(kek_id, CachedKek{.key = kek, .unwrapped_at_ms = now}); + return kek; +} + +Result<std::vector<uint8_t>> StandardEncryptionManager::DecryptManifestListKeyMetadata( + std::string_view manifest_list_key_id) { + std::lock_guard lock(mutex_); + auto it = keys_.find(std::string(manifest_list_key_id)); + if (it == keys_.end()) { + return Invalid("Cannot find manifest list key metadata with id {}", + manifest_list_key_id); + } + const EncryptedKey& manifest_list_key = it->second; + if (!manifest_list_key.encrypted_by_id.has_value() || + *manifest_list_key.encrypted_by_id == table_key_id_) { + return Invalid("{} is not manifest list key metadata", manifest_list_key_id); + } + + const std::string& kek_id = *manifest_list_key.encrypted_by_id; + ICEBERG_ASSIGN_OR_RAISE(auto kek, UnwrapKek(kek_id)); + auto timestamp = keys_.at(kek_id).properties.find(std::string(kKeyTimestamp)); + if (timestamp == keys_.at(kek_id).properties.end()) { + return Invalid("Key encryption key {} must be timestamped", kek_id); + } + + // The KEK timestamp is the AAD, to prevent timestamp tampering attacks. + ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(kek)); + return cipher->Decrypt(AsBytes(manifest_list_key.encrypted_key_metadata), + AsBytes(timestamp->second)); +} + +Result<std::string> StandardEncryptionManager::CurrentKekId() { + int64_t now = NowMs(); + for (const auto& key_id : key_order_) { + const EncryptedKey& key = keys_.at(key_id); + if (key.encrypted_by_id != table_key_id_) { + continue; + } + auto timestamp = ParseTimestamp(key); + if (timestamp.has_value() && now - *timestamp < kKekLifespan.count()) { + return key_id; + } + } + + // No unexpired key encryption key: create one, wrapped by the table master key. + std::vector<uint8_t> kek(data_key_length_); + ICEBERG_RETURN_UNEXPECTED(SecureRandomBytes(kek)); + ICEBERG_ASSIGN_OR_RAISE(auto wrapped, kms_->WrapKey(kek, table_key_id_)); + ICEBERG_ASSIGN_OR_RAISE(auto kek_id, GenerateKeyId()); + AddKey(EncryptedKey{.key_id = kek_id, + .encrypted_key_metadata = ToString(wrapped), + .encrypted_by_id = table_key_id_, + .properties = {{std::string(kKeyTimestamp), std::to_string(now)}}}); + unwrapped_keks_.insert_or_assign( + kek_id, CachedKek{.key = std::move(kek), .unwrapped_at_ms = now}); + return kek_id; +} + +Result<FileEncryptionKeys> StandardEncryptionManager::RegisterManifestListKeyMetadata( + const StandardKeyMetadata& key_metadata) { + ICEBERG_PRECHECK(key_metadata.file_length.has_value(), + "Manifest list key metadata must include the file length"); + std::lock_guard lock(mutex_); + ICEBERG_ASSIGN_OR_RAISE(auto kek_id, CurrentKekId()); + ICEBERG_ASSIGN_OR_RAISE(auto kek, UnwrapKek(kek_id)); + const EncryptedKey& kek_entry = keys_.at(kek_id); + const std::string& timestamp = kek_entry.properties.at(std::string(kKeyTimestamp)); + + ICEBERG_ASSIGN_OR_RAISE(auto cipher, AesGcmCipher::Make(kek)); + ICEBERG_ASSIGN_OR_RAISE(auto encrypted, + cipher->Encrypt(key_metadata.Serialize(), AsBytes(timestamp))); + ICEBERG_ASSIGN_OR_RAISE(auto file_key_id, GenerateKeyId()); + EncryptedKey file_key{.key_id = file_key_id, + .encrypted_key_metadata = ToString(encrypted), + .encrypted_by_id = kek_id}; + AddKey(file_key); + return FileEncryptionKeys{.key_encryption_key = keys_.at(kek_id), + .file_key = std::move(file_key)}; +} + +Result<std::unique_ptr<InputFile>> StandardEncryptionManager::Decrypt( + std::unique_ptr<EncryptedInputFile> encrypted) { + ICEBERG_PRECHECK(encrypted != nullptr, "Encrypted input file cannot be null"); + std::vector<uint8_t> key_metadata = encrypted->key_metadata().buffer(); + return NewDecryptingInputFile(encrypted->ReleaseInputFile(), key_metadata); Review Comment: Please preserve the native-format path instead of always constructing an AGS1 reader. Native Parquet metadata without `file_length` is rejected here, and adding the length still selects the wrong reader. Java returns Native wrappers with raw-file/key access and a separate AGS1 stream path. `Encrypt` and the new Native wrappers need the same two-path contract. ########## src/iceberg/encryption/standard_encryption_manager.h: ########## @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/standard_encryption_manager.h +/// Envelope encryption manager, compatible with Java's StandardEncryptionManager. + +#include <chrono> +#include <memory> +#include <mutex> +#include <string> +#include <unordered_map> +#include <vector> + +#include "iceberg/encryption/encryption_manager.h" +#include "iceberg/encryption/standard_key_metadata.h" +#include "iceberg/iceberg_bundle_export.h" + +namespace iceberg { + +/// \brief The two keys produced by registering a manifest list's key metadata. +struct ICEBERG_BUNDLE_EXPORT FileEncryptionKeys { + /// The key encryption key, wrapped by the table master key in the KMS. + EncryptedKey key_encryption_key; + /// The manifest list key metadata, encrypted by the key encryption key. + EncryptedKey file_key; +}; + +/// \brief Standard Iceberg envelope encryption. +/// +/// Key hierarchy: +/// - table master key: in the KMS, ID in table property `encryption.key-id`; +/// - key encryption keys (KEK): wrapped by the KMS and kept in `encryption-keys` +/// with `encrypted-by-id` = the table key ID and a KEY_TIMESTAMP property. A KEK is +/// used for 730 days (NIST SP 800-57), then a new one is created; +/// - manifest list key metadata: AES-GCM encrypted by a KEK with the KEK timestamp as +/// AAD, kept in `encryption-keys` and referenced by the snapshot `key-id`; +/// - manifest key metadata: in the manifest list `key_metadata` field; +/// - data and delete file key metadata: in the manifest `key_metadata` field. +/// +/// Unwrapped KEKs are cached for one hour. Thread-safe. +class ICEBERG_BUNDLE_EXPORT StandardEncryptionManager : public EncryptionManager { + public: + static constexpr std::string_view kKeyTimestamp = "KEY_TIMESTAMP"; + static constexpr int32_t kAadPrefixLength = 16; + static constexpr std::chrono::milliseconds kKekLifespan{730LL * 24 * 3600 * 1000}; + static constexpr std::chrono::milliseconds kUnwrappedKekCacheExpiry{3600LL * 1000}; + + static Result<std::shared_ptr<StandardEncryptionManager>> Make( + EncryptionManagerRegistry::Options options); + + // EncryptionManager + Result<std::unique_ptr<InputFile>> Decrypt( + std::unique_ptr<EncryptedInputFile> encrypted) override; + Result<std::unique_ptr<EncryptedOutputFile>> Encrypt( + std::unique_ptr<OutputFile> plain) override; Review Comment: These single-file overrides hide the batch overloads from `EncryptionManager`. A caller using the concrete pointer returned by `Make` cannot call `Encrypt` or `Decrypt` with a vector. Please expose the base overloads in both `StandardEncryptionManager` and `PlaintextEncryptionManager`, and cover calls through concrete and base pointers. ########## src/iceberg/encryption/encryption_key_metadata.h: ########## @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/encryption/encryption_key_metadata.h +/// Opaque, serialized key metadata, mirroring Java's EncryptionKeyMetadata. + +#include <cstdint> +#include <optional> +#include <utility> +#include <vector> + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +/// \brief Opaque serialized key metadata for one file. +/// +/// Implementations are free to choose any serialization; EncryptionManager and its +/// callers treat the result as an opaque byte buffer. +class ICEBERG_EXPORT EncryptionKeyMetadata { + public: + virtual ~EncryptionKeyMetadata() = default; + + /// \brief The serialized form of this key metadata. + /// + /// Implementations must not cache this value if their fields can change after + /// construction; it is recomputed on every call where that matters. + virtual std::vector<uint8_t> buffer() const = 0; +}; + +/// \brief EncryptionKeyMetadata backed by a fixed, already-serialized buffer. +class ICEBERG_EXPORT BaseEncryptionKeyMetadata : public EncryptionKeyMetadata { + public: + explicit BaseEncryptionKeyMetadata(std::vector<uint8_t> buffer) + : buffer_(std::move(buffer)) {} + + std::vector<uint8_t> buffer() const override { return buffer_; } + + private: + std::vector<uint8_t> buffer_; +}; + +/// \brief Key metadata for the native (AES-GCM) encryption key parameters. +/// +/// Exposes the data encryption key and AAD prefix directly, for formats (such as +/// Parquet) that perform their own native encryption rather than using the AGS1 stream +/// format. +class ICEBERG_EXPORT NativeEncryptionKeyMetadata : public EncryptionKeyMetadata { + public: + /// \brief The data encryption key. + virtual const std::vector<uint8_t>& EncryptionKey() const = 0; + + /// \brief The AAD prefix (file ID) used to bind ciphertext to this file, if any. + virtual const std::optional<std::vector<uint8_t>>& AadPrefix() const = 0; Review Comment: The Native metadata contract is still missing `FileLength` and `CopyWithLength`. Java writers use `CopyWithLength` after closing the file, while keeping the same key and AAD. Please support that flow through the interface. The base metadata also needs explicit empty and copy semantics; an empty vector cannot distinguish absent metadata from present empty bytes. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
