This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new eae924b chore: prepare Apache Paimon C++ 0.2.3 release (#155)
eae924b is described below
commit eae924b1e992cdd96d7a10bdbaf9bc82d5e536ff
Author: Zhang Jiawei <[email protected]>
AuthorDate: Thu Jul 23 21:19:09 2026 +0800
chore: prepare Apache Paimon C++ 0.2.3 release (#155)
* docs: align feature guidance with implementation
* docs: clarify build and commit integration
* chore: prepare Apache Paimon C++ 0.2.3 release
---
.github/workflows/build_release.yaml | 4 -
.github/workflows/clang_test.yaml | 1 -
.github/workflows/gcc8_test.yaml | 4 +-
.github/workflows/gcc_test.yaml | 2 -
.github/workflows/test_with_sanitizer.yaml | 2 -
.gitignore | 1 +
CMakeLists.txt | 2 +-
CONTRIBUTING.md | 1 -
README.md | 7 +-
docs/source/_static/versions.json | 6 +-
docs/source/api/file_format.rst | 4 -
docs/source/basic_concepts.rst | 10 +-
docs/source/build_system.rst | 55 +++--
docs/source/building.rst | 11 +-
docs/source/conf.py | 2 +-
docs/source/index.rst | 2 +-
docs/source/user_guide/arrow.rst | 4 +-
docs/source/user_guide/clean.rst | 2 +-
docs/source/user_guide/commit.rst | 8 +-
docs/source/user_guide/data_types.rst | 8 +-
docs/source/user_guide/prefetch.rst | 44 +++-
docs/source/user_guide/write.rst | 77 ++++---
examples/CMakeLists.txt | 21 +-
examples/clean_demo.cpp | 15 +-
include/paimon/commit_message.h | 10 +-
include/paimon/defs.h | 5 +-
include/paimon/file_store_commit.h | 5 +
.../paimon/global_index/global_index_write_task.h | 6 +-
.../paimon/global_index/global_indexer_factory.h | 2 +-
scripts/releasing/README.md | 81 ++++++++
scripts/releasing/create_source_release.sh | 180 ++++++++++++++++
scripts/releasing/verify_release_candidate.sh | 227 +++++++++++++++++++++
32 files changed, 687 insertions(+), 122 deletions(-)
diff --git a/.github/workflows/build_release.yaml
b/.github/workflows/build_release.yaml
index fe25c7a..0bed7ce 100644
--- a/.github/workflows/build_release.yaml
+++ b/.github/workflows/build_release.yaml
@@ -41,8 +41,6 @@ jobs:
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
@@ -64,8 +62,6 @@ jobs:
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/clang_test.yaml
b/.github/workflows/clang_test.yaml
index 20efbfa..ea30689 100644
--- a/.github/workflows/clang_test.yaml
+++ b/.github/workflows/clang_test.yaml
@@ -42,7 +42,6 @@ jobs:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
with:
- lfs: true
fetch-depth: 0 # fetch all history for git diff in clang-tidy
- name: Setup ccache
uses: ./.github/actions/setup-ccache
diff --git a/.github/workflows/gcc8_test.yaml b/.github/workflows/gcc8_test.yaml
index 405ffb0..01ed551 100644
--- a/.github/workflows/gcc8_test.yaml
+++ b/.github/workflows/gcc8_test.yaml
@@ -45,7 +45,7 @@ jobs:
- name: Install dependencies
run: |
apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8
ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential
python3-dev gdb sudo
+ DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8
ninja-build git tar curl tzdata zip unzip pkg-config build-essential
python3-dev gdb sudo
curl -L -O
https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local
--strip-components=1
rm cmake-3.28.3-linux-x86_64.tar.gz
@@ -60,8 +60,6 @@ jobs:
ls -la
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/gcc_test.yaml b/.github/workflows/gcc_test.yaml
index 8d23c7e..b213f66 100644
--- a/.github/workflows/gcc_test.yaml
+++ b/.github/workflows/gcc_test.yaml
@@ -41,8 +41,6 @@ jobs:
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.github/workflows/test_with_sanitizer.yaml
b/.github/workflows/test_with_sanitizer.yaml
index 06436eb..4005c8a 100644
--- a/.github/workflows/test_with_sanitizer.yaml
+++ b/.github/workflows/test_with_sanitizer.yaml
@@ -41,8 +41,6 @@ jobs:
steps:
- name: Checkout paimon-cpp
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 #
v6.0.1
- with:
- lfs: true
- name: Setup ccache
uses: ./.github/actions/setup-ccache
with:
diff --git a/.gitignore b/.gitignore
index a700cb5..b71dc56 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,6 +20,7 @@ build
build-release
build-debug
output
+release
# IDE settings
.idea
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 1726103..b6aca77 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT CMAKE_BUILD_TYPE)
endif()
project(paimon
- VERSION 0.2.0
+ VERSION 0.2.3
DESCRIPTION "Paimon C++ Project")
string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_BUILD_TYPE)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5aa4531..086f4a6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -59,7 +59,6 @@ Before submitting, please verify:
- **C++17** compatible compiler (GCC recommended)
- **CMake** ≥ 3.16
- **Python 3** (for linting scripts and pre-commit)
-- **git-lfs** (the repository uses Git LFS for large files)
---
diff --git a/README.md b/README.md
index 44b926e..57e32d4 100644
--- a/README.md
+++ b/README.md
@@ -49,14 +49,9 @@ The current implementation supports the `x86_64`
architecture.
## Building
-> **Note:** The build system and source files are being migrated
incrementally. The instructions below will work once the CMake build files and
source code are available in this repository.
-
-If you do not have `git-lfs` installed, install it first.
-
```bash
git clone https://github.com/apache/paimon-cpp.git
cd paimon-cpp
-git lfs pull
```
Build with CMake:
@@ -80,7 +75,7 @@ cp devcontainer.json.template devcontainer.json
## Collaboration
Paimon-cpp is an active open-source project and we welcome people who want to
contribute or share good ideas!
-Before contributing, please read the [Contributing Guide](CONTRIBUTING.md) and
the [Code Style Guide](docs/code-style.md). You are encouraged to check out our
[documentation](https://alibaba.github.io/paimon-cpp/).
+Before contributing, please read the [Contributing Guide](CONTRIBUTING.md) and
the [Code Style Guide](docs/code-style.md). You are encouraged to check out our
[documentation](https://paimon.apache.org/docs/cpp/).
## License
diff --git a/docs/source/_static/versions.json
b/docs/source/_static/versions.json
index 9980db1..28cf190 100644
--- a/docs/source/_static/versions.json
+++ b/docs/source/_static/versions.json
@@ -1,7 +1,7 @@
[
{
- "name": "0.10.0 (dev)",
- "version": "dev/",
- "url": "#"
+ "name": "0.2.3",
+ "version": "0.2.3",
+ "url": "https://paimon.apache.org/docs/cpp/";
}
]
diff --git a/docs/source/api/file_format.rst b/docs/source/api/file_format.rst
index 57b9eee..23c88cc 100644
--- a/docs/source/api/file_format.rst
+++ b/docs/source/api/file_format.rst
@@ -36,10 +36,6 @@ Interface
:members:
:undoc-members:
-.. doxygenclass:: paimon::BatchReader
- :members:
- :undoc-members:
-
.. doxygenclass:: paimon::FileBatchReader
:members:
:undoc-members:
diff --git a/docs/source/basic_concepts.rst b/docs/source/basic_concepts.rst
index 0cf87f4..84b8b5e 100644
--- a/docs/source/basic_concepts.rst
+++ b/docs/source/basic_concepts.rst
@@ -54,11 +54,8 @@ the corresponding snapshot.
Data Files
---------------------------
-Data files are grouped by partitions. Currently, Paimon supports using parquet
-(default) and orc as data file’s format.
-
-.. note::
- avro write as a data file format is not supported yet.
+Data files are grouped by partitions. Paimon C++ supports Parquet (default),
+ORC, and Avro as data file formats.
Partition
---------------------------
@@ -84,6 +81,3 @@ For any two writers modifying a table at the same time, as
long as they do not
modify the same partition, their commits can occur in parallel. If they modify
the same partition, only snapshot isolation is guaranteed. That is, the final
table state may be a mix of the two commits, but no changes are lost.
-
-.. note::
- Paimon C++ currently does not support compaction.
diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst
index 8bcf714..f3b18e5 100644
--- a/docs/source/build_system.rst
+++ b/docs/source/build_system.rst
@@ -29,7 +29,7 @@ as in-memory data format interface. Please ensure that Arrow
C++ is installed
and available to your build system
The recommended way to integrate the Paimon C++ libraries into your C++ project
-is to use CMake’s `find_package
<https://cmake.org/cmake/help/latest/command/find_package.html>`_
+is to use CMake's `find_package
<https://cmake.org/cmake/help/latest/command/find_package.html>`_
function to locate and integrate dependencies.
CMake
@@ -48,11 +48,27 @@ for data format and file system.
project(MyExample)
- find_package(Arrow REQUIRED)
- find_package(Paimon REQUIRED)
+ # Arrow's static package may export LZ4 under a different target name.
+ find_package(lz4 CONFIG QUIET)
+ if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS LZ4::lz4_static)
+ elseif(TARGET lz4::lz4 AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS lz4::lz4)
+ endif()
+
+ find_package(Arrow CONFIG REQUIRED)
+ find_package(Paimon CONFIG REQUIRED)
+
+ if(TARGET Arrow::arrow_shared)
+ set(PAIMON_ARROW_TARGET Arrow::arrow_shared)
+ elseif(TARGET Arrow::arrow_static)
+ set(PAIMON_ARROW_TARGET Arrow::arrow_static)
+ else()
+ message(FATAL_ERROR "No supported Arrow CMake target is available")
+ endif()
add_executable(my_example my_example.cc)
- target_link_libraries(my_example PRIVATE Arrow::arrow_shared
+ target_link_libraries(my_example PRIVATE ${PAIMON_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_local_file_system_shared)
@@ -64,12 +80,15 @@ The directive ``find_package(Paimon REQUIRED)`` instructs
CMake to locate a
Paimon C++ installation on your system. If successful, it sets ``Paimon_FOUND``
to true if the Paimon C++ libraries were found.
-It also defines the following linkable imported targets:
+It defines the following supported imported target:
* ``Paimon::paimon_shared`` links to the Paimon shared libraries
-* ``Paimon::paimon_static`` links to the Paimon static libraries
-In most cases, it is recommended to use the Paimon shared libraries.
+Static targets such as ``Paimon::paimon_static`` may also be present when
+Paimon is built with ``PAIMON_BUILD_STATIC=ON``. They are not currently
+supported as standalone installed targets because their exported link
+interfaces do not include every third-party dependency. Installed consumers
+should use the shared targets.
Optional plugins (built-in file formats, file systems, and index)
-----------------------------------------------------------------
@@ -78,22 +97,24 @@ Paimon provides a set of built-in optional plugins that you
can link to as neede
- File format plugins:
- - ``Paimon::paimon_parquet_file_format_shared`` /
``Paimon::paimon_parquet_file_format_static``
- - ``Paimon::paimon_orc_file_format_shared`` /
``Paimon::paimon_orc_file_format_static``
- - ``Paimon::paimon_avro_file_format_shared`` /
``Paimon::paimon_avro_file_format_static``
- - ``Paimon::paimon_blob_file_format_shared`` /
``Paimon::paimon_blob_file_format_static``
+ - ``Paimon::paimon_parquet_file_format_shared``
+ - ``Paimon::paimon_orc_file_format_shared``
+ - ``Paimon::paimon_avro_file_format_shared``
+ - ``Paimon::paimon_blob_file_format_shared``
- File system plugins:
- - ``Paimon::paimon_local_file_system_shared`` /
``Paimon::paimon_local_file_system_static``
- - ``Paimon::paimon_jindo_file_system_shared`` /
``Paimon::paimon_jindo_file_system_static``
+ - ``Paimon::paimon_local_file_system_shared``
+ - ``Paimon::paimon_jindo_file_system_shared``
- Index plugins:
- - ``Paimon::paimon_file_index_shared`` / ``Paimon::paimon_file_index_static``
- - ``Paimon::paimon_lumina_index_shared`` /
``Paimon::paimon_lumina_index_static``
- - ``Paimon::paimon_lucene_index_shared`` /
``Paimon::paimon_lucene_index_static``
+ - ``Paimon::paimon_file_index_shared``
+ - ``Paimon::paimon_lumina_index_shared``
+ - ``Paimon::paimon_lucene_index_shared``
.. note::
- In most cases, it is recommended to use the shared variants of these plugins.
+ Static plugin targets have the same installed-consumer limitation as
+ ``Paimon::paimon_static`` and should not be used as standalone imported
+ targets.
diff --git a/docs/source/building.rst b/docs/source/building.rst
index 71df728..8b106c0 100644
--- a/docs/source/building.rst
+++ b/docs/source/building.rst
@@ -60,7 +60,6 @@ repository:
$ git clone https://github.com/apache/paimon-cpp.git
$ cd paimon-cpp
- $ git lfs pull
Manual configuration
--------------------
@@ -160,14 +159,22 @@ require project-specific patches, so their supported
source values are
-Dzstd_SOURCE=BUNDLED
Use ``PAIMON_PACKAGE_PREFIX`` to provide one common prefix for dependencies
-whose own ``<Package>_ROOT`` variable is not set.
+whose own ``<Package>_ROOT`` variable is not set. Because the patched Arrow and
+ORC dependencies cannot be resolved from the system, a global ``SYSTEM`` build
+must override them to ``BUNDLED``:
.. code-block:: shell
cmake -B build \
-DPAIMON_DEPENDENCY_SOURCE=SYSTEM \
+ -DArrow_SOURCE=BUNDLED \
+ -DORC_SOURCE=BUNDLED \
-DPAIMON_PACKAGE_PREFIX=/opt/paimon-deps
+All other enabled dependencies must be available as system packages or under
+the specified prefix. When ORC support is disabled, the ``ORC_SOURCE`` override
+can be omitted.
+
Package-manager-specific modes are intentionally out of scope for this first
dependency source interface. They can still be used through standard CMake
mechanisms such as ``CMAKE_PREFIX_PATH`` or ``CMAKE_TOOLCHAIN_FILE``, while
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5789bbc..989ea07 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -114,7 +114,7 @@ html_sidebars = {
# The master toctree document.
master_doc = "index"
-version = "0.2.0"
+version = "0.2.3"
html_theme_options = {
"show_toc_level": 2,
diff --git a/docs/source/index.rst b/docs/source/index.rst
index c595d45..5c4d83a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -86,7 +86,7 @@ native engines to access the Paimon datalake format with
maximum efficiency.
:class-card: contrib-card
:shadow: none
- Explore Paimon‘s API reference documentation
+ Explore Paimon's API reference documentation
+++
diff --git a/docs/source/user_guide/arrow.rst b/docs/source/user_guide/arrow.rst
index 084b156..a2e0229 100644
--- a/docs/source/user_guide/arrow.rst
+++ b/docs/source/user_guide/arrow.rst
@@ -62,7 +62,7 @@ costs and increasing long-term maintenance complexity.
Adopting the Arrow C Data Interface
-----------------------------------
-To leverage Arrow’s performance and ecosystem benefits while avoiding tight
+To leverage Arrow's performance and ecosystem benefits while avoiding tight
coupling to specific Arrow C++ SDK versions, we use the Arrow C Data
Interface as the default in-memory format for Paimon C++.
@@ -114,7 +114,7 @@ Memory management
Nullability and validity
~~~~~~~~~~~~~~~~~~~~~~~~
-- Use standard validity bitmaps for nullability and adhere to Arrow’s
canonical buffer organization (validity, offsets, data, etc.).
+- Use standard validity bitmaps for nullability and adhere to Arrow's
canonical buffer organization (validity, offsets, data, etc.).
Conversion boundaries
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/user_guide/clean.rst b/docs/source/user_guide/clean.rst
index 05e2645..32099dd 100644
--- a/docs/source/user_guide/clean.rst
+++ b/docs/source/user_guide/clean.rst
@@ -85,7 +85,7 @@ Detailed Steps
1. Build a ``ScanFilter`` for the partition and use the latest snapshot
to scan the partition.
2. Iterate over the scanned data file list (``ManifestEntries``) and
- rewrite each entry’s type to ``DELETE``.
+ rewrite each entry's type to ``DELETE``.
3. Commit using the rewritten ``ManifestEntries``. If the commit fails,
retry a limited number of times.
diff --git a/docs/source/user_guide/commit.rst
b/docs/source/user_guide/commit.rst
index e833991..7e7ee17 100644
--- a/docs/source/user_guide/commit.rst
+++ b/docs/source/user_guide/commit.rst
@@ -18,7 +18,7 @@
Commit
==========================================
-Commit is a critical stage in Paimon’s write path. It is responsible for
generating
+Commit is a critical stage in Paimon's write path. It is responsible for
generating
Snapshot files that describe the current state of a Paimon table. This document
provides a detailed analysis of the Paimon Commit process.
@@ -44,7 +44,7 @@ Collect File Changes
~~~~~~~~~~~~~~~~~~~~~~~~
During Commit, the system extracts key information from
``CommitMessages``—such as
-file name, operation type (``ADD`` or ``DELETE``), the file’s Partition and
Bucket—
+file name, operation type (``ADD`` or ``DELETE``), the file's Partition and
Bucket—
and converts them into ``ManifestEntry`` records.
A ``ManifestEntry`` represents a single operation record in a manifest file and
@@ -83,7 +83,7 @@ classifies them as follows:
Minor Compaction
^^^^^^^^^^^^^^^^
-If Full Compaction’s conditions are not met, Minor Compaction is attempted:
+If Full Compaction's conditions are not met, Minor Compaction is attempted:
- The system iterates over all files, skipping any file larger than the target
file size.
- Whenever the accumulated size of selected files exceeds the target file
size, those files are merged.
@@ -101,7 +101,7 @@ Generate the Base Manifest List
After compaction (which may or may not be triggered every time), the system
obtains
a consolidated set of manifest file metadata. This metadata is written into a
new
-manifest list file, forming the Snapshot’s base manifest list.
+manifest list file, forming the Snapshot's base manifest list.
Generate New Manifest Files and the Delta Manifest List
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/source/user_guide/data_types.rst
b/docs/source/user_guide/data_types.rst
index 5759119..de794d8 100644
--- a/docs/source/user_guide/data_types.rst
+++ b/docs/source/user_guide/data_types.rst
@@ -150,8 +150,9 @@ and `Arrow DataTypes
<https://arrow.apache.org/docs/format/Columnar.html#data-ty
- Data type of a timestamp without time zone consisting of year-month-day
hour:minute:second[.fractional] with up to nanosecond precision and values
ranging from 0000-01-01 00:00:00.000000000 to 9999-12-31 23:59:59.999999999.
The type can be declared using ``TIMESTAMP(p)`` where p is the number of
- digits of fractional seconds (precision). p must have a value between 0
- and 9 (both inclusive). If no precision is specified, p is equal to 6.
+ digits of fractional seconds (precision). Paimon C++ supports precision
+ values 0 (seconds), 3 (milliseconds), 6 (microseconds), and 9
+ (nanoseconds). If no precision is specified, p is equal to 6.
* - ``TIMESTAMP WITH LOCAL TIME ZONE``
@@ -164,6 +165,9 @@ and `Arrow DataTypes
<https://arrow.apache.org/docs/format/Columnar.html#data-ty
to the configured session time zone. A conversion from and to int
describes
the number of seconds since epoch. A conversion from and to long
describes the number of milliseconds since epoch.
+ Paimon C++ supports precision values 0 (seconds), 3 (milliseconds), 6
+ (microseconds), and 9 (nanoseconds).
+
* - ``ARRAY<t>``
- List
- Data type of an array of elements with same subtype.
diff --git a/docs/source/user_guide/prefetch.rst
b/docs/source/user_guide/prefetch.rst
index 15ae9d7..9f62da2 100644
--- a/docs/source/user_guide/prefetch.rst
+++ b/docs/source/user_guide/prefetch.rst
@@ -23,26 +23,50 @@ Prefetch
:align: center
:width: 100%
-In C++ Paimon, we use a multi-producer, single-consumer model to optimize file
reading. The core idea is to split a file into line-based ReadRanges and assign
them to multiple reader threads (producers). Each reader thread owns an
independent result queue that holds its processed RecordBatches. In the main
reader thread (the consumer), we sort the heads of all queues by the ReadRange
start offset in ascending order and select the RecordBatch with the smallest
start offset to ensure globa [...]
+In C++ Paimon, we use a multi-producer, single-consumer model to optimize file
+reading. The core idea is to split a file into line-based ReadRanges and assign
+them to multiple reader threads (producers). Each reader thread owns an
+independent result queue that holds its processed RecordBatches. In the main
+reader thread (the consumer), we sort the heads of all queues by the ReadRange
+start offset in ascending order and select the RecordBatch with the smallest
+start offset to ensure globally ordered results.
Read Range Splitting Strategy
=============================
-Designing an efficient ReadRange splitting strategy requires balancing two key
objectives:
+Designing an efficient ReadRange splitting strategy requires balancing two key
+objectives:
-- Minimize read amplification: Ensure the data fetched from storage is used
effectively, avoiding unnecessary I/O overhead.
-- Reduce ReadRange span: Ideally, the size of a ReadRange should match a
single read batch size to enable fine-grained parallel control.
+- Minimize read amplification: Ensure the data fetched from storage is used
+ effectively, avoiding unnecessary I/O overhead.
+- Reduce ReadRange span: Ideally, the size of a ReadRange should match a single
+ read batch size to enable fine-grained parallel control.
-Below we detail how these strategies are applied to formats Parquet.
+Below we detail how these strategies are applied to Parquet and ORC.
Parquet
========
-Parquet files are organized into RowGroups and Pages. Since C++ Parquet does
not support row-level seeking, prefetching can only be done at the RowGroup
level. This naturally avoids read amplification, but introduces a new
challenge: if a file contains only a small number of RowGroups, parallelism is
severely limited. Therefore, we recommend users reduce RowGroup size when
writing Parquet files to increase opportunities for parallel processing.
+Parquet files are organized into RowGroups and Pages. Since C++ Parquet does
+not support row-level seeking, prefetching can only be done at the RowGroup
+level. This naturally avoids read amplification, but introduces a new
+challenge: if a file contains only a small number of RowGroups, parallelism is
+severely limited. Therefore, we recommend users reduce RowGroup size when
+writing Parquet files to increase opportunities for parallel processing.
-Another critical difference is the read behavior compared to Orc. Orc strictly
returns RecordBatches aligned to Stripe boundaries, whereas C++ Parquet may
return a RecordBatch containing data from multiple RowGroups. This can lead to
output order confusion during parallel reads. We modified C++ Parquet internals
to return results strictly aligned to RowGroup boundaries, matching Orc’s
behavior. With this change, parallel reading no longer requires complex seek
operations, improving overa [...]
+Another critical difference is the read behavior compared to ORC. ORC strictly
+returns RecordBatches aligned to Stripe boundaries, whereas C++ Parquet may
+return a RecordBatch containing data from multiple RowGroups. This can lead to
+output order confusion during parallel reads. We modified C++ Parquet internals
+to return results strictly aligned to RowGroup boundaries, matching ORC's
+behavior. With this change, parallel reading no longer requires complex seek
+operations, improving overall read efficiency.
-.. admonition:: TODO
- :class: tip
+ORC
+===
- Support prefetch for Orc.
+ORC files are organized into Stripes. Paimon C++ generates row ranges from
+Stripe metadata, the selected columns, and the configured natural read size.
+Prefetch is enabled when the estimated compressed size of the selected columns
+exceeds the configured threshold; otherwise the reader uses the normal
+single-reader path.
diff --git a/docs/source/user_guide/write.rst b/docs/source/user_guide/write.rst
index da2120b..6fd5c41 100644
--- a/docs/source/user_guide/write.rst
+++ b/docs/source/user_guide/write.rst
@@ -17,14 +17,14 @@
Write
=====
-Batch writing requires the compute engine to pre-bucket data (bucket), using
the
-same bucketing strategy as Paimon to ensure correct ``Scan`` behavior, and to
-specify the target ``partition``. Data should be accumulated into
``RecordBatch``
-and written to Paimon.
+Batch writing requires the compute engine to specify the target ``partition``.
+For fixed-bucket tables, the engine must also assign a valid bucket to every
+``RecordBatch``. In unaware-bucket and postpone-bucket modes, the writer can
+resolve the bucket automatically when it is omitted.
Paimon C++ uses Apache Arrow as the :ref:`in-memory columnar
format<memory-format>`
-to more efficiently support writing to disk columnar formats such as ORC and
-Parquet, thereby improving write throughput.
+to more efficiently support writing to disk columnar formats such as ORC,
+Parquet, and Avro, thereby improving write throughput.
.. note::
Currently supported table types:
@@ -33,14 +33,13 @@ Parquet, thereby improving write throughput.
Not supported in the current scope:
- Changelog
- - Indexes
Bucketing Modes
---------------
- Append tables:
- * Support ``bucket = -1`` (dynamic bucket mode)
+ * Support ``bucket = -1`` (unaware-bucket mode)
* Support ``bucket > 0`` (fixed bucket mode)
- PK tables:
@@ -56,9 +55,15 @@ RecordBatch Construction
- The compute engine must:
- - Apply the Paimon-consistent bucketing function to each row prior to
batching.
- Assign the correct ``partition`` for each row.
- - Group rows into Arrow ``RecordBatch`` per partition-bucket combination to
minimize writer state changes and I/O overhead.
+ - In fixed-bucket mode, apply the Paimon-consistent bucketing function, set a
+ bucket in ``[0, bucket)``, and group rows into Arrow ``RecordBatch``
objects
+ per partition-bucket combination.
+
+- In unaware-bucket mode (append table with ``bucket = -1``), an omitted bucket
+ is resolved to ``0``.
+- In postpone-bucket mode (primary-key table with ``bucket = -2``), an omitted
+ bucket is resolved to ``-2``.
- Recommended practices:
@@ -70,7 +75,7 @@ Prepare Commit
----------------
The compute engine is responsible for triggering the writer nodes'
``PrepareCommit``.
-Triggering conditions depend on the engine’s business needs and can follow
either:
+Triggering conditions depend on the engine's business needs and can follow
either:
- Streaming mode: time-based or periodic triggers (e.g., every N seconds).
- Batch mode: trigger after all data in the batch has been written.
@@ -98,7 +103,7 @@ public interfaces. To preserve compatibility without leaking
internal row
representations, Paimon C++ provides ``CommitMessage`` only through:
- Serialization: convert the internal commit state into a well-defined binary
- representation that matches Java Paimon’s expectations.
+ representation that matches Java Paimon's expectations.
- Deserialization: parse the Java-compatible binary representation back into
C++ commit structures for validation, replay, or tooling needs.
@@ -122,26 +127,27 @@ produce a correct ``Snapshot``, which commonly includes
(but is not limited to):
.. note::
- Current C++ scope supports Append and PK tables. Changelog is out of
- scope and should not be emitted in ``CommitMessage`` until
- explicitly supported.
+ The C++ writer supports Append and PK tables and can produce
+ ``CommitMessage`` objects for both. ``FileStoreCommit`` currently executes
+ local commits only for append-only tables on non-object-store file systems.
+ PK and object-store commit messages must be sent to an external control
+ plane. Changelog is out of scope and should not be emitted in
+ ``CommitMessage`` until explicitly supported.
Serialization and Deserialization
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-- Binary Format:
- - The binary payload must strictly conform to Java Paimon’s
``CommitMessage`` encoding.
- - Version tags or schema identifiers should be included to enable
forwards/backwards compatibility and safe upgrades.
-
-- Serialization API:
- - Provide a function to serialize the writer’s commit state into a byte
buffer (or stream) consumable by Java Paimon.
-
-- Deserialization API:
- - Provide a function to parse a Java-produced ``CommitMessage`` binary
payload back into C++ commit structures for verification, replay, and testing.
-
-- Validation:
- - Include conformance tests to assert that C++ serialized payloads are
accepted by Java Paimon.
- - Include round-trip tests to ensure C++ can parse Java-produced payloads
and vice versa for supported message versions.
+- **Binary format:** The binary payload must strictly conform to Java Paimon's
+ ``CommitMessage`` encoding. It does not contain a version tag, so callers
+ must transport ``CommitMessage::CurrentVersion()`` separately and supply it
+ when deserializing.
+- **Serialization API:** Use ``CommitMessage::Serialize`` for one message or
+ ``CommitMessage::SerializeList`` for a list.
+- **Deserialization API:** Use ``CommitMessage::Deserialize`` or
+ ``CommitMessage::DeserializeList`` with the separately supplied
+ serialization version.
+- **Validation:** Conformance and round-trip tests must verify compatibility
+ with Java Paimon for supported message versions.
Operational Flow
~~~~~~~~~~~~~~~~~~~~~~~
@@ -155,12 +161,15 @@ Operational Flow
3. Each writer invokes ``PrepareCommit``, which:
- Aggregates per-writer state into a ``CommitMessage``.
- - Serializes the message into a Java-compatible binary payload.
+ - Returns ``CommitMessage`` objects; it does not serialize them.
-4. The compute engine gathers ``CommitMessages`` from all writers.
+4. The compute engine gathers ``CommitMessage`` objects from all writers. For
+ cross-process transport, it explicitly calls ``Serialize`` or
+ ``SerializeList`` and carries ``CurrentVersion()`` alongside the payload.
-5. The compute engine issues a ``Commit`` request to the control plane with the
- collected messages, resulting in a new ``Snapshot``.
+5. For a supported local append-table commit, the engine passes the objects to
+ ``FileStoreCommit``. For PK tables or object-store paths, it sends the
+ serialized payload and version to an external control plane.
-6. The coordinator validates the messages, updates manifests/metadata, and
- finalizes the snapshot atomically.
+6. The local committer or external coordinator validates the messages, updates
+ manifests/metadata, and finalizes the snapshot atomically.
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 4fe7a8b..385b482 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -20,23 +20,40 @@ project(example)
set(CMAKE_CXX_STANDARD 17)
+# Arrow's static export refers to LZ4::lz4, while some lz4 packages export a
+# differently named target. Define a compatibility alias before loading Arrow.
+find_package(lz4 CONFIG QUIET)
+if(TARGET LZ4::lz4_static AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS LZ4::lz4_static)
+elseif(TARGET lz4::lz4 AND NOT TARGET LZ4::lz4)
+ add_library(LZ4::lz4 ALIAS lz4::lz4)
+endif()
+
find_package(Arrow CONFIG REQUIRED)
find_package(Paimon CONFIG REQUIRED)
+if(TARGET Arrow::arrow_shared)
+ set(PAIMON_EXAMPLE_ARROW_TARGET Arrow::arrow_shared)
+elseif(TARGET Arrow::arrow_static)
+ set(PAIMON_EXAMPLE_ARROW_TARGET Arrow::arrow_static)
+else()
+ message(FATAL_ERROR "Neither Arrow::arrow_shared nor Arrow::arrow_static
is available")
+endif()
+
add_executable(read_write_demo read_write_demo.cpp)
add_executable(clean_demo clean_demo.cpp)
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--no-as-needed ${CMAKE_EXE_LINKER_FLAGS}")
target_link_libraries(read_write_demo
- PRIVATE Arrow::arrow_shared
+ PRIVATE ${PAIMON_EXAMPLE_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_orc_file_format_shared
Paimon::paimon_local_file_system_shared)
target_link_libraries(clean_demo
- PRIVATE Arrow::arrow_shared
+ PRIVATE ${PAIMON_EXAMPLE_ARROW_TARGET}
Paimon::paimon_shared
Paimon::paimon_parquet_file_format_shared
Paimon::paimon_orc_file_format_shared
diff --git a/examples/clean_demo.cpp b/examples/clean_demo.cpp
index d34cb83..b2888a8 100644
--- a/examples/clean_demo.cpp
+++ b/examples/clean_demo.cpp
@@ -106,17 +106,26 @@ int main(int argc, char* argv[]) {
std::string temp_table_path = std::string(argv[2]);
std::string clean_mode = std::string(argv[3]);
+ if (clean_mode != "orphan_file" && clean_mode != "drop_partition" &&
+ clean_mode != "expire_snapshot") {
+ std::cerr << "Unknown clean mode: " << clean_mode
+ << ". Expected orphan_file, drop_partition, or
expire_snapshot." << std::endl;
+ return -1;
+ }
+
if (!CopyToTempDirectory(origin_table_path, temp_table_path)) {
return -1;
}
- std::map<std::string, std::string> clean_options;
paimon::Status status;
if (clean_mode == "orphan_file") {
std::cout << "enter the timestamp (ms) before which orphan files will
be deleted"
<< std::endl;
- int64_t older_than_ms;
- std::cin >> older_than_ms;
+ int64_t older_than_ms = 0;
+ if (!(std::cin >> older_than_ms)) {
+ std::cerr << "Invalid timestamp: expected an integer in
milliseconds." << std::endl;
+ return -1;
+ }
status = paimon::CleanOrphanFiles(temp_table_path, older_than_ms);
} else if (clean_mode == "drop_partition") {
std::cout << "enter partition key-value pairs to drop. type 'EOF EOF'
to finish"
diff --git a/include/paimon/commit_message.h b/include/paimon/commit_message.h
index 3d8757a..4521892 100644
--- a/include/paimon/commit_message.h
+++ b/include/paimon/commit_message.h
@@ -32,8 +32,12 @@ namespace paimon {
class CommitMessageSerializer;
class MemoryPool;
-/// Commit message for partition and bucket. Support Serialize and
Deserialize, compatible with java
-/// version.
+/// Commit message for partition and bucket. Supports serialization and
deserialization compatible
+/// with the Java version.
+///
+/// @note Serialized payloads do not embed their serialization version.
Transport
+/// `CurrentVersion()` alongside the payload and pass it explicitly to
`Deserialize()` or
+/// `DeserializeList()`.
// TODO(yonghao.fyh): to add some statistics of write (e.g., write bytes)
class PAIMON_EXPORT CommitMessage {
public:
@@ -44,6 +48,7 @@ class PAIMON_EXPORT CommitMessage {
/// Serializes a single commit message to a binary string format.
/// The serialized format is compatible with the Java version of Paimon.
+ /// The serialization version is not included in the returned payload.
/// @param commit_message The commit message to serialize.
/// @param pool Memory pool for memory allocation during serialization.
/// @return Result containing the serialized string data, or an error if
serialization fails.
@@ -51,6 +56,7 @@ class PAIMON_EXPORT CommitMessage {
const std::shared_ptr<MemoryPool>&
pool);
/// Serializes a list of commit messages to a binary string format.
+ /// The serialization version is not included in the returned payload.
/// @param commit_messages Vector of commit messages to serialize.
/// @param pool Memory pool for memory allocation during serialization.
/// @return Result containing the serialized string data, or an error if
serialization fails.
diff --git a/include/paimon/defs.h b/include/paimon/defs.h
index c15256c..c38cb49 100644
--- a/include/paimon/defs.h
+++ b/include/paimon/defs.h
@@ -92,8 +92,9 @@ struct PAIMON_EXPORT Options {
static const char SEQUENCE_GROUP[];
/// @}
- /// "bucket" - Bucket number for file store. It should either be equal to
-1 (dynamic bucket
- /// mode), or it must be greater than 0 (fixed bucket mode).
+ /// "bucket" - Bucket mode or bucket count for file store. Append tables
support -1
+ /// (unaware-bucket mode), primary-key tables support -2 (postpone-bucket
mode), and both
+ /// table types support values greater than 0 (fixed-bucket mode).
static const char BUCKET[];
/// "bucket-key" - Specify the paimon distribution policy. Data is
assigned to each bucket
diff --git a/include/paimon/file_store_commit.h
b/include/paimon/file_store_commit.h
index a702c41..66d3d39 100644
--- a/include/paimon/file_store_commit.h
+++ b/include/paimon/file_store_commit.h
@@ -43,6 +43,11 @@ class CommitMessage;
///
/// The `FileStoreCommit` class provides interfaces for committing changes,
expiring old snapshots,
/// dropping partitions, and retrieving commit metrics.
+///
+/// @note Local commit execution currently supports append-only tables on
non-object-store file
+/// systems. `Create()` returns `NotImplemented` for primary-key tables and
object-store paths.
+/// Primary-key writers can still produce `CommitMessage` objects; those
messages must be committed
+/// by an external control plane.
class PAIMON_EXPORT FileStoreCommit {
public:
/// Create an instance of `FileStoreCommit`.
diff --git a/include/paimon/global_index/global_index_write_task.h
b/include/paimon/global_index/global_index_write_task.h
index 6151382..a66f7ab 100644
--- a/include/paimon/global_index/global_index_write_task.h
+++ b/include/paimon/global_index/global_index_write_task.h
@@ -40,10 +40,10 @@ class PAIMON_EXPORT GlobalIndexWriteTask {
/// @param table_path Path to the table root directory where index files
are stored.
/// @param field_name Name of the indexed column (must be present in the
table schema).
/// @param index_type Type of global index to build (e.g., "bitmap",
"lumina").
- /// @param index_split The indexed split containing the actual data
(e.g., Parquet file) and
- // row id range [from, to] for data to build index.
+ /// @param indexed_split The indexed split containing the actual data
(e.g., Parquet file) and
+ /// row id range [from, to] for data to build index.
/// The range must be fully contained within the data
covered
- /// by the given `split`.
+ /// by the given `indexed_split`.
/// @param options Index-specific configuration (e.g., false positive
rate for bloom
/// filters).
/// @param pool Memory pool for temporary allocations during index
construction.
diff --git a/include/paimon/global_index/global_indexer_factory.h
b/include/paimon/global_index/global_indexer_factory.h
index 6b0783a..e3d3069 100644
--- a/include/paimon/global_index/global_indexer_factory.h
+++ b/include/paimon/global_index/global_indexer_factory.h
@@ -52,7 +52,7 @@ class PAIMON_EXPORT GlobalIndexerFactory : public Factory {
static Result<std::unique_ptr<GlobalIndexer>> Get(
const std::string& identifier, const std::map<std::string,
std::string>& options);
- /// Creates a `GlobalIndexer` using the current factory’s implementation
and the given options.
+ /// Creates a `GlobalIndexer` using the current factory's implementation
and the given options.
virtual Result<std::unique_ptr<GlobalIndexer>> Create(
const std::map<std::string, std::string>& options) const = 0;
};
diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md
new file mode 100644
index 0000000..bf9b7dd
--- /dev/null
+++ b/scripts/releasing/README.md
@@ -0,0 +1,81 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Apache Paimon C++ release scripts
+
+These scripts create and verify the source artifact voted on by the Apache
Paimon
+PMC. They do not publish artifacts, create tags, or move files between Apache
+distribution repositories.
+
+## Create a release candidate
+
+Create and push a signed RC tag before creating the source artifact:
+
+```bash
+git tag -s release-0.2.3-rc1 -m "Apache Paimon C++ 0.2.3 RC1"
+git push upstream release-0.2.3-rc1
+```
+
+Create the source artifact, SHA-512 checksum, and detached OpenPGP signature:
+
+```bash
+scripts/releasing/create_source_release.sh \
+ --version 0.2.3 \
+ --git-ref release-0.2.3-rc1 \
+ --output-dir release/0.2.3-rc1 \
+ --signing-key ASF_GPG_KEY_ID
+```
+
+The output files are:
+
+```text
+apache-paimon-cpp-0.2.3-src.tgz
+apache-paimon-cpp-0.2.3-src.tgz.asc
+apache-paimon-cpp-0.2.3-src.tgz.sha512
+```
+
+Existing files are never overwritten. A changed candidate must use a new RC
+directory and a new vote.
+
+## Verify a release candidate
+
+Download the source artifact, its `.asc` and `.sha512` files, and the Paimon
+`KEYS` file. Import `KEYS` into a temporary or dedicated GPG keyring, then run:
+
+```bash
+RAT_JAR=/path/to/apache-rat-0.16.1.jar \
+ scripts/releasing/verify_release_candidate.sh \
+ apache-paimon-cpp-0.2.3-src.tgz
+```
+
+The verifier checks:
+
+- the SHA-512 checksum and detached OpenPGP signature;
+- archive paths and the single `paimon-cpp-0.2.3/` root directory;
+- required `LICENSE`, `NOTICE`, build, and documentation files;
+- the CMake and documentation versions;
+- absence of common compiled artifact types;
+- Apache RAT results;
+- a release build and the test suite from the extracted source archive.
+
+The `--allow-unsigned` and `--skip-rat` options are only for local development
+of the release process. A voter may use `--skip-build` when the repository's
+Linux CI build command is not suitable for their platform, but must then build
+and test the extracted source distribution separately before casting a binding
+vote.
diff --git a/scripts/releasing/create_source_release.sh
b/scripts/releasing/create_source_release.sh
new file mode 100755
index 0000000..fb0345a
--- /dev/null
+++ b/scripts/releasing/create_source_release.sh
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+SOURCE_ROOT=$(cd "${SCRIPT_DIR}/../.." && pwd)
+
+RELEASE_VERSION=""
+GIT_REF=""
+OUTPUT_DIR="${SOURCE_ROOT}/release"
+SIGNING_KEY=""
+
+usage() {
+ cat <<'EOF'
+Create an Apache Paimon C++ source release from an immutable Git ref.
+
+Usage:
+ create_source_release.sh --version VERSION --git-ref REF [options]
+
+Required:
+ --version VERSION Release version, for example 0.2.3
+ --git-ref REF Commit or signed RC tag to archive
+
+Options:
+ --output-dir DIR Output directory (default: <repository>/release)
+ --signing-key KEY_ID Create an ASCII-armored detached OpenPGP signature
+ -h, --help Show this help
+
+The script creates:
+ apache-paimon-cpp-VERSION-src.tgz
+ apache-paimon-cpp-VERSION-src.tgz.sha512
+ apache-paimon-cpp-VERSION-src.tgz.asc (when --signing-key is provided)
+
+Existing artifacts are never overwritten.
+EOF
+}
+
+fail() {
+ echo "Error: $*" >&2
+ exit 1
+}
+
+calculate_sha512() {
+ local file=$1
+ if command -v sha512sum >/dev/null 2>&1; then
+ sha512sum "${file}" | awk '{print $1}'
+ elif command -v shasum >/dev/null 2>&1; then
+ shasum -a 512 "${file}" | awk '{print $1}'
+ else
+ fail "sha512sum or shasum is required"
+ fi
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --version)
+ [[ $# -ge 2 ]] || fail "--version requires a value"
+ RELEASE_VERSION=$2
+ shift 2
+ ;;
+ --git-ref)
+ [[ $# -ge 2 ]] || fail "--git-ref requires a value"
+ GIT_REF=$2
+ shift 2
+ ;;
+ --output-dir)
+ [[ $# -ge 2 ]] || fail "--output-dir requires a value"
+ OUTPUT_DIR=$2
+ shift 2
+ ;;
+ --signing-key)
+ [[ $# -ge 2 ]] || fail "--signing-key requires a value"
+ SIGNING_KEY=$2
+ shift 2
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ fail "unknown argument: $1"
+ ;;
+ esac
+done
+
+[[ -n "${RELEASE_VERSION}" ]] || fail "--version is required"
+[[ -n "${GIT_REF}" ]] || fail "--git-ref is required"
+[[ "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] ||
+ fail "release version must use MAJOR.MINOR.PATCH format"
+
+git -C "${SOURCE_ROOT}" rev-parse --verify "${GIT_REF}^{commit}" >/dev/null
2>&1 ||
+ fail "Git ref does not resolve to a commit: ${GIT_REF}"
+
+for required_file in CMakeLists.txt LICENSE NOTICE; do
+ git -C "${SOURCE_ROOT}" cat-file -e "${GIT_REF}:${required_file}"
2>/dev/null ||
+ fail "${required_file} is missing from ${GIT_REF}"
+done
+
+CMAKE_VERSION=$(
+ git -C "${SOURCE_ROOT}" show "${GIT_REF}:CMakeLists.txt" |
+ sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' |
+ head -n 1
+)
+[[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "CMake version ${CMAKE_VERSION:-<missing>} does not match
${RELEASE_VERSION}"
+
+DOCS_VERSION=$(
+ git -C "${SOURCE_ROOT}" show "${GIT_REF}:docs/source/conf.py" |
+ sed -n 's/^version = "\([^"]*\)"$/\1/p' |
+ head -n 1
+)
+[[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "documentation version ${DOCS_VERSION:-<missing>} does not match
${RELEASE_VERSION}"
+
+ARTIFACT_NAME="apache-paimon-cpp-${RELEASE_VERSION}-src.tgz"
+ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}"
+
+mkdir -p "${OUTPUT_DIR}"
+OUTPUT_DIR=$(cd "${OUTPUT_DIR}" && pwd)
+
+for suffix in "" ".sha512" ".asc"; do
+ [[ ! -e "${OUTPUT_DIR}/${ARTIFACT_NAME}${suffix}" ]] ||
+ fail "refusing to overwrite ${OUTPUT_DIR}/${ARTIFACT_NAME}${suffix}"
+done
+
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+
+echo "Creating ${ARTIFACT_NAME} from ${GIT_REF}..."
+git -C "${SOURCE_ROOT}" -c tar.umask=0022 archive \
+ --format=tar \
+ --prefix="${ARCHIVE_ROOT}/" \
+ "${GIT_REF}" |
+ gzip -n >"${TEMP_DIR}/${ARTIFACT_NAME}"
+
+SHA512=$(calculate_sha512 "${TEMP_DIR}/${ARTIFACT_NAME}")
+printf '%s %s\n' "${SHA512}" "${ARTIFACT_NAME}" \
+ >"${TEMP_DIR}/${ARTIFACT_NAME}.sha512"
+
+if [[ -n "${SIGNING_KEY}" ]]; then
+ command -v gpg >/dev/null 2>&1 || fail "gpg is required to sign the
artifact"
+ echo "Signing ${ARTIFACT_NAME} with ${SIGNING_KEY}..."
+ gpg --armor \
+ --local-user "${SIGNING_KEY}" \
+ --detach-sign \
+ --output "${TEMP_DIR}/${ARTIFACT_NAME}.asc" \
+ "${TEMP_DIR}/${ARTIFACT_NAME}"
+fi
+
+mv "${TEMP_DIR}/${ARTIFACT_NAME}" "${OUTPUT_DIR}/"
+mv "${TEMP_DIR}/${ARTIFACT_NAME}.sha512" "${OUTPUT_DIR}/"
+if [[ -n "${SIGNING_KEY}" ]]; then
+ mv "${TEMP_DIR}/${ARTIFACT_NAME}.asc" "${OUTPUT_DIR}/"
+fi
+
+echo "Created release artifacts in ${OUTPUT_DIR}:"
+echo " ${ARTIFACT_NAME}"
+echo " ${ARTIFACT_NAME}.sha512"
+if [[ -n "${SIGNING_KEY}" ]]; then
+ echo " ${ARTIFACT_NAME}.asc"
+else
+ echo " Signature not created; pass --signing-key for an official release
candidate."
+fi
diff --git a/scripts/releasing/verify_release_candidate.sh
b/scripts/releasing/verify_release_candidate.sh
new file mode 100755
index 0000000..cf6660d
--- /dev/null
+++ b/scripts/releasing/verify_release_candidate.sh
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+ARTIFACT=""
+RAT_JAR=${RAT_JAR:-}
+ALLOW_UNSIGNED=false
+SKIP_RAT=false
+SKIP_BUILD=false
+
+usage() {
+ cat <<'EOF'
+Verify an Apache Paimon C++ source release candidate.
+
+Usage:
+ verify_release_candidate.sh [options] ARTIFACT
+
+Options:
+ --rat-jar FILE Apache RAT executable jar (or set RAT_JAR)
+ --allow-unsigned Allow a missing .asc file for local preparation only
+ --skip-rat Skip Apache RAT for local preparation only
+ --skip-build Skip the release build and test suite
+ -h, --help Show this help
+
+By default the script requires:
+ ARTIFACT.sha512
+ ARTIFACT.asc
+ an Apache RAT jar
+
+It verifies the checksum and signature, inspects and extracts the archive,
+checks release metadata, runs Apache RAT, then builds and tests from the
+extracted source distribution.
+EOF
+}
+
+fail() {
+ echo "Error: $*" >&2
+ exit 1
+}
+
+calculate_sha512() {
+ local file=$1
+ if command -v sha512sum >/dev/null 2>&1; then
+ sha512sum "${file}" | awk '{print $1}'
+ elif command -v shasum >/dev/null 2>&1; then
+ shasum -a 512 "${file}" | awk '{print $1}'
+ else
+ fail "sha512sum or shasum is required"
+ fi
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --rat-jar)
+ [[ $# -ge 2 ]] || fail "--rat-jar requires a value"
+ RAT_JAR=$2
+ shift 2
+ ;;
+ --allow-unsigned)
+ ALLOW_UNSIGNED=true
+ shift
+ ;;
+ --skip-rat)
+ SKIP_RAT=true
+ shift
+ ;;
+ --skip-build)
+ SKIP_BUILD=true
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ -*)
+ fail "unknown argument: $1"
+ ;;
+ *)
+ [[ -z "${ARTIFACT}" ]] || fail "only one artifact may be specified"
+ ARTIFACT=$1
+ shift
+ ;;
+ esac
+done
+
+[[ -n "${ARTIFACT}" ]] || fail "an artifact is required"
+[[ -f "${ARTIFACT}" ]] || fail "artifact does not exist: ${ARTIFACT}"
+
+ARTIFACT_DIR=$(cd "$(dirname "${ARTIFACT}")" && pwd)
+ARTIFACT_NAME=$(basename "${ARTIFACT}")
+ARTIFACT="${ARTIFACT_DIR}/${ARTIFACT_NAME}"
+
+if [[ "${ARTIFACT_NAME}" =~
^apache-paimon-cpp-([0-9]+\.[0-9]+\.[0-9]+)-src\.tgz$ ]]; then
+ RELEASE_VERSION=${BASH_REMATCH[1]}
+else
+ fail "unexpected artifact name: ${ARTIFACT_NAME}"
+fi
+
+CHECKSUM_FILE="${ARTIFACT}.sha512"
+SIGNATURE_FILE="${ARTIFACT}.asc"
+ARCHIVE_ROOT="paimon-cpp-${RELEASE_VERSION}"
+
+[[ -f "${CHECKSUM_FILE}" ]] || fail "missing checksum: ${CHECKSUM_FILE}"
+EXPECTED_SHA512=$(awk 'NR == 1 {print $1}' "${CHECKSUM_FILE}")
+CHECKSUM_ARTIFACT=$(awk 'NR == 1 {print $2}' "${CHECKSUM_FILE}")
+[[ "${EXPECTED_SHA512}" =~ ^[0-9a-fA-F]{128}$ ]] ||
+ fail "invalid SHA-512 file: ${CHECKSUM_FILE}"
+[[ "${CHECKSUM_ARTIFACT}" == "${ARTIFACT_NAME}" ]] ||
+ fail "checksum file names ${CHECKSUM_ARTIFACT:-<missing>}, expected
${ARTIFACT_NAME}"
+ACTUAL_SHA512=$(calculate_sha512 "${ARTIFACT}")
+EXPECTED_SHA512=$(printf '%s' "${EXPECTED_SHA512}" | tr '[:upper:]'
'[:lower:]')
+ACTUAL_SHA512=$(printf '%s' "${ACTUAL_SHA512}" | tr '[:upper:]' '[:lower:]')
+[[ "${ACTUAL_SHA512}" == "${EXPECTED_SHA512}" ]] ||
+ fail "SHA-512 checksum does not match"
+echo "SHA-512 checksum: valid"
+
+if [[ -f "${SIGNATURE_FILE}" ]]; then
+ command -v gpg >/dev/null 2>&1 || fail "gpg is required to verify the
signature"
+ gpg --verify "${SIGNATURE_FILE}" "${ARTIFACT}"
+ echo "OpenPGP signature: valid"
+elif [[ "${ALLOW_UNSIGNED}" == true ]]; then
+ echo "OpenPGP signature: skipped for local preparation"
+else
+ fail "missing signature: ${SIGNATURE_FILE}"
+fi
+
+TEMP_DIR=$(mktemp -d)
+trap 'rm -rf "${TEMP_DIR}"' EXIT
+CONTENTS_FILE="${TEMP_DIR}/archive-contents.txt"
+
+tar -tzf "${ARTIFACT}" >"${CONTENTS_FILE}"
+[[ -s "${CONTENTS_FILE}" ]] || fail "source archive is empty"
+
+while IFS= read -r entry; do
+ [[ "${entry}" != /* ]] || fail "archive contains an absolute path:
${entry}"
+ [[ "${entry}" != ".." && "${entry}" != ../* && "${entry}" != */.. &&
+ "${entry}" != *"/../"* ]] ||
+ fail "archive contains path traversal: ${entry}"
+ [[ "${entry}" == "${ARCHIVE_ROOT}" || "${entry}" == "${ARCHIVE_ROOT}/"* ]]
||
+ fail "archive entry is outside ${ARCHIVE_ROOT}: ${entry}"
+done <"${CONTENTS_FILE}"
+
+tar -xzf "${ARTIFACT}" -C "${TEMP_DIR}"
+SOURCE_DIR="${TEMP_DIR}/${ARCHIVE_ROOT}"
+[[ -d "${SOURCE_DIR}" ]] || fail "archive root is missing: ${ARCHIVE_ROOT}"
+
+for required_file in LICENSE NOTICE CMakeLists.txt docs/source/conf.py; do
+ [[ -f "${SOURCE_DIR}/${required_file}" ]] ||
+ fail "required release file is missing: ${required_file}"
+done
+
+CMAKE_VERSION=$(
+ sed -n 's/^[[:space:]]*VERSION[[:space:]]\+\([0-9][0-9.]*\).*$/\1/p' \
+ "${SOURCE_DIR}/CMakeLists.txt" |
+ head -n 1
+)
+[[ "${CMAKE_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "CMake version ${CMAKE_VERSION:-<missing>} does not match
${RELEASE_VERSION}"
+
+DOCS_VERSION=$(
+ sed -n 's/^version = "\([^"]*\)"$/\1/p'
"${SOURCE_DIR}/docs/source/conf.py" |
+ head -n 1
+)
+[[ "${DOCS_VERSION}" == "${RELEASE_VERSION}" ]] ||
+ fail "documentation version ${DOCS_VERSION:-<missing>} does not match
${RELEASE_VERSION}"
+
+UNEXPECTED_BINARIES=$(
+ find "${SOURCE_DIR}" -type f \
+ \( -name '*.a' -o -name '*.class' -o -name '*.dll' -o -name '*.dylib' \
+ -o -name '*.exe' -o -name '*.jar' -o -name '*.lib' -o -name '*.o' \
+ -o -name '*.pdb' -o -name '*.pyc' -o -name '*.so' \) \
+ -print
+)
+[[ -z "${UNEXPECTED_BINARIES}" ]] ||
+ fail "source archive contains unexpected compiled
files:${UNEXPECTED_BINARIES}"
+
+UNSAFE_PERMISSIONS=$(
+ find "${SOURCE_DIR}" -type f \( -perm -0020 -o -perm -0002 \) -print
+)
+[[ -z "${UNSAFE_PERMISSIONS}" ]] ||
+ fail "source archive contains group- or world-writable
files:${UNSAFE_PERMISSIONS}"
+
+echo "Archive layout and release metadata: valid"
+
+if [[ "${SKIP_RAT}" == false ]]; then
+ [[ -n "${RAT_JAR}" ]] || fail "--rat-jar or RAT_JAR is required"
+ [[ -f "${RAT_JAR}" ]] || fail "Apache RAT jar does not exist: ${RAT_JAR}"
+
+ RAT_REPORT="${TEMP_DIR}/rat-report.txt"
+ java -jar "${RAT_JAR}" \
+ -E "${SOURCE_DIR}/.github/.rat-excludes" \
+ -d "${SOURCE_DIR}" >"${RAT_REPORT}"
+
+ if grep -Eq 'Files with unapproved licenses:[[:space:]]*[1-9]'
"${RAT_REPORT}"; then
+ cat "${RAT_REPORT}"
+ fail "Apache RAT found files with unapproved licenses"
+ fi
+ echo "Apache RAT: valid"
+else
+ echo "Apache RAT: skipped for local preparation"
+fi
+
+if [[ "${SKIP_BUILD}" == false ]]; then
+ "${SOURCE_DIR}/ci/scripts/build_paimon.sh" "${SOURCE_DIR}" false false
Release
+ echo "Release build and tests: valid"
+else
+ echo "Release build and tests: skipped"
+fi
+
+echo "Release candidate verification completed successfully."
