This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 1b6d6a3f build: enable Parquet modular encryption and zstd in vendored
Arrow (#989)
1b6d6a3f is described below
commit 1b6d6a3f85bf9d8f0ffcbe59a67e59310bac6580
Author: Xinli Shang <[email protected]>
AuthorDate: Thu Oct 15 13:32:26 2026 +0900
build: enable Parquet modular encryption and zstd in vendored Arrow (#989)
Part of #988.
Adds the `ICEBERG_ENCRYPTION` CMake option (default on), which builds the
vendored Arrow with Parquet modular encryption and zstd and links OpenSSL into
the bundle library. This makes the crypto available for the following table
encryption steps; there is no behavior change for tables.
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.github/workflows/aws_test.yml | 2 +-
.github/workflows/cpp-linter.yml | 2 +-
.github/workflows/rc.yml | 2 +-
.github/workflows/sanitizer_test.yml | 2 +-
.github/workflows/sql_catalog_test.yml | 2 +-
.github/workflows/test.yml | 15 ++--
CMakeLists.txt | 2 +
cmake_modules/IcebergThirdpartyToolchain.cmake | 55 +++++++++++++++
src/iceberg/CMakeLists.txt | 10 +++
src/iceberg/test/CMakeLists.txt | 3 +
src/iceberg/test/parquet_encryption_test.cc | 97 ++++++++++++++++++++++++++
11 files changed, 180 insertions(+), 12 deletions(-)
diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index 08de48a7..51357046 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -87,7 +87,7 @@ jobs:
- name: Install dependencies on Ubuntu
if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
shell: bash
- run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libjitterentropy3-dev
+ run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libssl-dev libjitterentropy3-dev
- name: Cache vcpkg packages
if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk
== 'OFF' }}
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml
index 5ca6c4b1..be59a610 100644
--- a/.github/workflows/cpp-linter.yml
+++ b/.github/workflows/cpp-linter.yml
@@ -60,7 +60,7 @@ jobs:
shell: bash
run: |
sudo apt-get update
- sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev
libpq-dev default-libmysqlclient-dev
+ sudo apt-get install -y libcurl4-openssl-dev libssl-dev
libsqlite3-dev libpq-dev default-libmysqlclient-dev
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
index ba494db4..6cbcbeac 100644
--- a/.github/workflows/rc.yml
+++ b/.github/workflows/rc.yml
@@ -104,7 +104,7 @@ jobs:
- name: Install dependencies on Ubuntu
if: ${{ startsWith(matrix.os, 'ubuntu') }}
shell: bash
- run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev
+ run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libssl-dev
- name: Verify
run: |
diff --git a/.github/workflows/sanitizer_test.yml
b/.github/workflows/sanitizer_test.yml
index 343e661e..98612c6c 100644
--- a/.github/workflows/sanitizer_test.yml
+++ b/.github/workflows/sanitizer_test.yml
@@ -51,7 +51,7 @@ jobs:
persist-credentials: false
- name: Install dependencies
shell: bash
- run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev
+ run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libssl-dev
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
diff --git a/.github/workflows/sql_catalog_test.yml
b/.github/workflows/sql_catalog_test.yml
index 91b5d60e..28ddaae3 100644
--- a/.github/workflows/sql_catalog_test.yml
+++ b/.github/workflows/sql_catalog_test.yml
@@ -96,7 +96,7 @@ jobs:
if: ${{ startsWith(matrix.runs-on, 'windows') &&
steps.vcpkg-cache.outputs.cache-hit != 'true' }}
shell: pwsh
run: |
- vcpkg install zlib:x64-windows nlohmann-json:x64-windows
nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
+ vcpkg install zlib:x64-windows nlohmann-json:x64-windows
nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
openssl:x64-windows
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index eb289f07..a85e6f24 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -53,10 +53,10 @@ jobs:
matrix:
include:
- cmake_build_type: Debug
- extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON"
+ extra_cmake_args: "-DICEBERG_CXX20_COMPAT=ON
-DICEBERG_ENCRYPTION=ON"
api_mode: C++20 Compat
- cmake_build_type: RelWithDebInfo
- extra_cmake_args: ""
+ extra_cmake_args: "-DICEBERG_ENCRYPTION=ON"
api_mode: C++23
steps:
- name: Checkout iceberg-cpp
@@ -65,7 +65,7 @@ jobs:
persist-credentials: false
- name: Install dependencies
shell: bash
- run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev
+ run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libssl-dev
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
@@ -130,7 +130,7 @@ jobs:
persist-credentials: false
- name: Install dependencies
shell: bash
- run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev
+ run: sudo apt-get update && sudo apt-get install -y
libcurl4-openssl-dev libssl-dev
- name: Restore sccache cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 #
v6.1.0
with:
@@ -179,7 +179,8 @@ jobs:
- name: Build Iceberg
shell: bash
env:
- ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON"
+ # Enables the Homebrew keg-only OpenSSL lookup, which only runs on
macOS.
+ ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON
-DICEBERG_ENCRYPTION=ON"
run: |
ci/scripts/build_iceberg.sh \
--source-dir "$(pwd)" \
@@ -231,7 +232,7 @@ jobs:
if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }}
shell: pwsh
run: |
- vcpkg install zlib:x64-windows nlohmann-json:x64-windows
nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows
+ vcpkg install zlib:x64-windows nlohmann-json:x64-windows
nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows openssl:x64-windows
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
@@ -239,7 +240,7 @@ jobs:
- name: Build Iceberg
shell: pwsh
env:
- ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON"
+ ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_CXX20_COMPAT=ON
-DICEBERG_ENCRYPTION=ON"
run: |
$ErrorActionPreference = "Stop"
bash -lc '
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 315af807..7a9c3b2f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -51,6 +51,8 @@ option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
option(ICEBERG_BUILD_TESTS "Build tests" ON)
option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF)
option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON)
+option(ICEBERG_ENCRYPTION
+ "Build table encryption support (Parquet modular encryption, AES GCM
streams)" OFF)
option(ICEBERG_BUILD_REST "Build rest catalog client" ON)
option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration
tests" OFF)
option(ICEBERG_BUILD_HIVE "Build hive (HMS) catalog client" OFF)
diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake
b/cmake_modules/IcebergThirdpartyToolchain.cmake
index aa1a88e0..fef63f65 100644
--- a/cmake_modules/IcebergThirdpartyToolchain.cmake
+++ b/cmake_modules/IcebergThirdpartyToolchain.cmake
@@ -163,7 +163,28 @@ endmacro()
# ----------------------------------------------------------------------
# Apache Arrow
+# Find OpenSSL for table encryption. Called before Arrow is configured, so
that the
+# imported OpenSSL targets are visible to the vendored Arrow subdirectories.
+macro(resolve_openssl_for_encryption)
+ if(APPLE
+ AND NOT DEFINED OPENSSL_ROOT_DIR
+ AND NOT DEFINED ENV{OPENSSL_ROOT_DIR})
+ # Homebrew's OpenSSL is keg-only and not on the default search path.
+ execute_process(COMMAND brew --prefix openssl@3
+ OUTPUT_VARIABLE ICEBERG_BREW_OPENSSL_PREFIX
+ OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+ if(ICEBERG_BREW_OPENSSL_PREFIX AND EXISTS "${ICEBERG_BREW_OPENSSL_PREFIX}")
+ set(OPENSSL_ROOT_DIR "${ICEBERG_BREW_OPENSSL_PREFIX}")
+ endif()
+ endif()
+ find_package(OpenSSL REQUIRED)
+endmacro()
+
function(resolve_arrow_dependency)
+ if(ICEBERG_ENCRYPTION)
+ resolve_openssl_for_encryption()
+ endif()
+
prepare_fetchcontent()
# Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via
@@ -178,12 +199,21 @@ function(resolve_arrow_dependency)
set(ARROW_S3 ${ICEBERG_S3})
set(ARROW_JSON ON)
set(ARROW_PARQUET ON)
+ if(ICEBERG_ENCRYPTION)
+ # Parquet modular encryption, used for encrypted Iceberg data and delete
files.
+ set(PARQUET_REQUIRE_ENCRYPTION ON)
+ endif()
set(ARROW_ENABLE_THREADING ON)
set(ARROW_SIMD_LEVEL "NONE")
set(ARROW_RUNTIME_SIMD_LEVEL "NONE")
set(ARROW_POSITION_INDEPENDENT_CODE ON)
set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
set(ARROW_WITH_ZLIB ON)
+ # zstd is the default value of the write.parquet.compression-codec table
property,
+ # so a vendored Arrow without it cannot write Iceberg's default Parquet
files.
+ # Not required by modular encryption itself; parquet_encryption_test covers
the
+ # encrypted-plus-zstd combination to keep the two features verified together.
+ set(ARROW_WITH_ZSTD ON)
if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED)
set(AWSSDK_SOURCE "SYSTEM")
endif()
@@ -217,6 +247,25 @@ function(resolve_arrow_dependency)
fetchcontent_makeavailable(VendoredArrow)
if(vendoredarrow_SOURCE_DIR)
+ # Arrow's Parquet targets do not inherit the OpenSSL include directory when
+ # Arrow is vendored; add it for the Parquet encryption sources. Depending
on the
+ # platform and build type, they are compiled in the object library or
directly in
+ # the static/shared library. The bundle library links OpenSSL::Crypto.
+ #
+ # Only vendored targets are touched here. With system Arrow the same names
can
+ # be ALIAS targets created by a parent project, and
target_include_directories
+ # cannot modify an ALIAS.
+ if(ICEBERG_ENCRYPTION)
+ foreach(parquet_target parquet_objlib parquet_static parquet_shared)
+ if(TARGET ${parquet_target})
+ get_target_property(parquet_target_alias ${parquet_target}
ALIASED_TARGET)
+ if(NOT parquet_target_alias)
+ target_include_directories(${parquet_target} PRIVATE
${OPENSSL_INCLUDE_DIR})
+ endif()
+ endif()
+ endforeach()
+ endif()
+
if(NOT TARGET Arrow::arrow_static)
add_library(Arrow::arrow_static INTERFACE IMPORTED)
target_link_libraries(Arrow::arrow_static INTERFACE arrow_static)
@@ -268,6 +317,12 @@ function(resolve_arrow_dependency)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Arrow Parquet)
endif()
+ # The bundle exports OpenSSL::Crypto in its link interface for both vendored
and
+ # system Arrow, so consumers must find OpenSSL in both modes.
+ if(ICEBERG_ENCRYPTION)
+ list(APPEND ICEBERG_SYSTEM_DEPENDENCIES OpenSSL)
+ endif()
+
set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
index 657d487c..6939b2be 100644
--- a/src/iceberg/CMakeLists.txt
+++ b/src/iceberg/CMakeLists.txt
@@ -360,6 +360,16 @@ if(ICEBERG_BUILD_BUNDLE)
"$<IF:$<BOOL:${AVRO_VENDORED}>,iceberg::avrocpp_s,$<IF:$<TARGET_EXISTS:avro-cpp::avrocpp_shared>,avro-cpp::avrocpp_shared,avro-cpp::avrocpp_static>>"
)
+ if(ICEBERG_ENCRYPTION)
+ foreach(libs
+ ICEBERG_BUNDLE_STATIC_BUILD_INTERFACE_LIBS
+ ICEBERG_BUNDLE_SHARED_BUILD_INTERFACE_LIBS
+ ICEBERG_BUNDLE_STATIC_INSTALL_INTERFACE_LIBS
+ ICEBERG_BUNDLE_SHARED_INSTALL_INTERFACE_LIBS)
+ list(APPEND ${libs} OpenSSL::Crypto)
+ endforeach()
+ endif()
+
add_iceberg_lib(iceberg_bundle
SOURCES
${ICEBERG_BUNDLE_SOURCES}
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index 659db4f6..3a38db35 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -221,6 +221,9 @@ if(ICEBERG_BUILD_BUNDLE)
parquet_metrics_test.cc
parquet_schema_test.cc
parquet_test.cc)
+ if(ICEBERG_ENCRYPTION)
+ target_sources(parquet_test PRIVATE parquet_encryption_test.cc)
+ endif()
add_iceberg_test(scan_test
USE_BUNDLE
diff --git a/src/iceberg/test/parquet_encryption_test.cc
b/src/iceberg/test/parquet_encryption_test.cc
new file mode 100644
index 00000000..10ec64f8
--- /dev/null
+++ b/src/iceberg/test/parquet_encryption_test.cc
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <memory>
+#include <string>
+
+#include <arrow/io/memory.h>
+#include <arrow/json/from_string.h>
+#include <arrow/table.h>
+#include <arrow/util/secure_string.h>
+#include <gtest/gtest.h>
+#include <parquet/arrow/reader.h>
+#include <parquet/arrow/writer.h>
+#include <parquet/encryption/encryption.h>
+#include <parquet/exception.h>
+#include <parquet/file_reader.h>
+#include <parquet/properties.h>
+
+namespace iceberg::parquet {
+
+namespace {
+
+const std::string kKey = "0123456789012345";
+const std::string kAadPrefix = "file-aad-prefix-";
+
+std::shared_ptr<::arrow::Table> MakeTable() {
+ auto schema = ::arrow::schema({::arrow::field("id", ::arrow::int64())});
+ auto ids =
+ ::arrow::json::ArrayFromJSONString(::arrow::int64(), "[1, 2,
3]").ValueOrDie();
+ return ::arrow::Table::Make(schema, {ids});
+}
+
+std::shared_ptr<::arrow::Buffer> WriteEncrypted(::parquet::Compression::type
codec) {
+ // Same settings as Iceberg: footer key only (encrypted footer), AAD prefix
supplied
+ // by the reader from the manifest instead of stored in the file.
+ ::parquet::FileEncryptionProperties::Builder encryption{
+ ::arrow::util::SecureString(std::string(kKey))};
+ encryption.aad_prefix(kAadPrefix);
+ encryption.disable_aad_prefix_storage();
+ auto properties = ::parquet::WriterProperties::Builder()
+ .compression(codec)
+ ->encryption(encryption.build())
+ ->build();
+
+ auto sink = ::arrow::io::BufferOutputStream::Create().ValueOrDie();
+ PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(
+ *MakeTable(), ::arrow::default_memory_pool(), sink, 1024, properties));
+ return sink->Finish().ValueOrDie();
+}
+
+std::shared_ptr<::arrow::Table> Read(const std::shared_ptr<::arrow::Buffer>&
buffer,
+ const std::string& aad_prefix) {
+ ::parquet::FileDecryptionProperties::Builder decryption;
+ decryption.footer_key(::arrow::util::SecureString(std::string(kKey)));
+ decryption.aad_prefix(aad_prefix);
+ ::parquet::ReaderProperties properties;
+ properties.file_decryption_properties(decryption.build());
+
+ auto reader = ::parquet::ParquetFileReader::Open(
+ std::make_shared<::arrow::io::BufferReader>(buffer), properties);
+ auto arrow_reader =
::parquet::arrow::FileReader::Make(::arrow::default_memory_pool(),
+ std::move(reader))
+ .ValueOrDie();
+ return arrow_reader->ReadTable().ValueOrDie();
+}
+
+} // namespace
+
+TEST(ParquetEncryptionTest, ModularEncryptionIsAvailable) {
+ for (auto codec :
+ {::parquet::Compression::UNCOMPRESSED, ::parquet::Compression::ZSTD}) {
+ auto buffer = WriteEncrypted(codec);
+ // Encrypted footer mode
+ ASSERT_EQ(std::string(reinterpret_cast<const char*>(buffer->data()), 4),
"PARE");
+ EXPECT_TRUE(Read(buffer, kAadPrefix)->Equals(*MakeTable()));
+ // The AAD prefix binds the file to its manifest entry
+ EXPECT_THROW(Read(buffer, "another-prefix--"),
::parquet::ParquetException);
+ }
+}
+
+} // namespace iceberg::parquet
