This is an automated email from the ASF dual-hosted git repository.
MisterRaindrop pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 1ada3f17 CI: support native extensions on Cloudberry 2.x and main
1ada3f17 is described below
commit 1ada3f179d51231df61ceeb3b8f068d2e1b13c10
Author: Jinbao Chen <[email protected]>
AuthorDate: Mon Sep 7 02:57:07 2026 -0400
CI: support native extensions on Cloudberry 2.x and main
Make the PXF native extensions compatible with both Cloudberry 2.x
(PostgreSQL 14) and Cloudberry main (PostgreSQL 16).
Use Node pointers for makeString() results so the same source works
before and after PostgreSQL split the Value node types, and include
arpa/inet.h for the byte-order helper declarations.
Extend the Rocky 9 CI workflow to:
- build Cloudberry RPMs for both REL_2_STABLE and main
- scope RPM caches and artifacts by Cloudberry ref and commit
- record and verify the Cloudberry build identity and PostgreSQL major
- build native extensions against both PostgreSQL 14 and 16
- force clean native builds in test containers to avoid reusing
incompatible host objects
Co-authored-by: Jinbao Chen <[email protected]>
Co-authored-by: liuxiaoyu <[email protected]>
---
.github/workflows/pxf-ci.yml | 142 +++++++++++++++++----
.../testcontainers/pxf-cbdb/script/build_pxf.sh | 10 +-
ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh | 53 +++++++-
external-table/src/gpdbwritableformatter.c | 1 +
external-table/src/pxfuriparser.c | 4 +-
external-table/test/pxfprotocol_test.c | 2 +-
fdw/pxf_option.c | 2 +-
7 files changed, 175 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml
index 3b82c269..5da1a314 100644
--- a/.github/workflows/pxf-ci.yml
+++ b/.github/workflows/pxf-ci.yml
@@ -119,15 +119,33 @@ jobs:
retention-days: 7
build-cloudberry-rpm:
- name: Build Cloudberry RPM Package
+ name: Build Cloudberry RPM Package (${{ matrix.cloudberry.id }})
runs-on: ubuntu-latest
container:
image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
options: --user root
+ strategy:
+ fail-fast: false
+ matrix:
+ cloudberry:
+ - id: cb2
+ ref: REL_2_STABLE
+ pg_major: '14'
+ - id: cb3
+ ref: main
+ pg_major: '16'
steps:
- - name: Get month number
- id: get-month
- run: echo "month=$(/bin/date -u '+%Y-%m')" >> "$GITHUB_OUTPUT"
+ - name: Checkout Cloudberry source
+ uses: actions/checkout@v4
+ with:
+ repository: apache/cloudberry
+ ref: ${{ matrix.cloudberry.ref }}
+ path: workspace/cloudberry
+ submodules: true
+
+ - name: Resolve Cloudberry commit
+ id: cloudberry
+ run: echo "sha=$(git -C workspace/cloudberry rev-parse HEAD)" >>
"$GITHUB_OUTPUT"
- name: Restore RPM cache
id: cache-rpm
@@ -135,17 +153,8 @@ jobs:
with:
path: |
workspace/cloudberry-rpm
- workspace/cloudberry-source-rocky9.tar.gz
- key: cloudberry-rpm-${{ runner.os }}-${{ steps.get-month.outputs.month
}}
-
- - name: Checkout Cloudberry source
- if: steps.cache-rpm.outputs.cache-hit != 'true'
- uses: actions/checkout@v4
- with:
- repository: apache/cloudberry
- ref: ${{ env.CLOUDBERRY_VERSION }}
- path: workspace/cloudberry
- submodules: true
+ workspace/cloudberry-source-rocky9-${{ matrix.cloudberry.id }}.tar.gz
+ key: cloudberry-rpm-rocky9-${{ matrix.cloudberry.id }}-${{
steps.cloudberry.outputs.sha }}
- name: Checkout PXF source (for build script)
if: steps.cache-rpm.outputs.cache-hit != 'true'
@@ -165,30 +174,38 @@ jobs:
if: steps.cache-rpm.outputs.cache-hit != 'true'
run: |
cd workspace
- tar czf cloudberry-source-rocky9.tar.gz cloudberry/
+ tar czf cloudberry-source-rocky9-${{ matrix.cloudberry.id }}.tar.gz
cloudberry/
+
+ - name: Record Cloudberry build identity
+ run: |
+ printf '%s\n' '${{ matrix.cloudberry.ref }}' >
workspace/cloudberry-rpm/cloudberry-build.ref
+ printf '%s\n' '${{ steps.cloudberry.outputs.sha }}' >
workspace/cloudberry-rpm/cloudberry-build.sha
+ printf '%s\n' '${{ matrix.cloudberry.pg_major }}' >
workspace/cloudberry-rpm/cloudberry-build.pg-major
- name: Save RPM cache
- # save cache from default branch only (this cache can be reused between
PRs)
+ # Save exact-ref caches from the default branch only; PRs may restore
them.
if: steps.cache-rpm.outputs.cache-hit != 'true' && github.ref ==
'refs/heads/main'
uses: actions/cache/save@v4
with:
path: |
workspace/cloudberry-rpm
- workspace/cloudberry-source-rocky9.tar.gz
- key: cloudberry-rpm-${{ runner.os }}-${{ steps.get-month.outputs.month
}}
+ workspace/cloudberry-source-rocky9-${{ matrix.cloudberry.id }}.tar.gz
+ key: cloudberry-rpm-rocky9-${{ matrix.cloudberry.id }}-${{
steps.cloudberry.outputs.sha }}
- name: Upload RPM artifact
uses: actions/upload-artifact@v4
with:
- name: cloudberry-rpm
- path: workspace/cloudberry-rpm/*.rpm
+ name: cloudberry-rpm-${{ matrix.cloudberry.id }}
+ path: |
+ workspace/cloudberry-rpm/*.rpm
+ workspace/cloudberry-rpm/cloudberry-build.*
retention-days: 7
- name: Upload Cloudberry source artifact (Rocky 9)
uses: actions/upload-artifact@v4
with:
- name: cloudberry-source-rocky9
- path: workspace/cloudberry-source-rocky9.tar.gz
+ name: cloudberry-source-rocky9-${{ matrix.cloudberry.id }}
+ path: workspace/cloudberry-source-rocky9-${{ matrix.cloudberry.id
}}.tar.gz
retention-days: 7
build-cloudberry-rpm-rocky10:
@@ -405,6 +422,67 @@ jobs:
path: pxf-cbdb-testcontainer-${{ matrix.distro }}.tar
retention-days: 1
+ native-extension-compatibility:
+ name: Native extensions (${{ matrix.cloudberry.id }})
+ needs: [build-cloudberry-rpm]
+ runs-on: ubuntu-latest
+ container:
+ image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
+ options: --user root
+ strategy:
+ fail-fast: false
+ matrix:
+ cloudberry:
+ - id: cb2
+ ref: REL_2_STABLE
+ pg_major: '14'
+ - id: cb3
+ ref: main
+ pg_major: '16'
+ steps:
+ - name: Checkout PXF source
+ uses: actions/checkout@v4
+
+ - name: Download Cloudberry RPM
+ uses: actions/download-artifact@v4
+ with:
+ name: cloudberry-rpm-${{ matrix.cloudberry.id }}
+ path: /tmp/cloudberry-rpm
+
+ - name: Verify Cloudberry build identity
+ run: |
+ test "$(cat /tmp/cloudberry-rpm/cloudberry-build.ref)" = '${{
matrix.cloudberry.ref }}'
+ test "$(cat /tmp/cloudberry-rpm/cloudberry-build.pg-major)" = '${{
matrix.cloudberry.pg_major }}'
+ echo "Cloudberry ref: $(cat /tmp/cloudberry-rpm/cloudberry-build.ref)"
+ echo "Cloudberry commit: $(cat
/tmp/cloudberry-rpm/cloudberry-build.sha)"
+
+ - name: Install Cloudberry RPM
+ run: |
+ mapfile -t packages < <(find /tmp/cloudberry-rpm -maxdepth 1 -type f
-name 'apache-cloudberry-db*.rpm' -print)
+ if [ "${#packages[@]}" -ne 1 ]; then
+ echo "Expected exactly one Cloudberry RPM, found ${#packages[@]}"
+ find /tmp/cloudberry-rpm -maxdepth 1 -type f -print
+ exit 1
+ fi
+ rpm -Uvh --force "${packages[0]}"
+
+ - name: Build native extensions
+ run: |
+ source /usr/local/cloudberry-db/cloudberry-env.sh
+ actual_version=$(pg_config --version)
+ actual_pg_major=${actual_version#* }
+ actual_pg_major=${actual_pg_major%%.*}
+ echo "Using $actual_version"
+ postgres --gp-version
+ if [ "$actual_pg_major" != '${{ matrix.cloudberry.pg_major }}' ]; then
+ echo "Expected PostgreSQL ${{ matrix.cloudberry.pg_major }}, got
$actual_version"
+ exit 1
+ fi
+ make -C external-table clean
+ make -C external-table
+ make -C fdw clean
+ make -C fdw
+
# Stage 2: Parallel test jobs using matrix strategy
pxf-test:
name: Test PXF - ${{ matrix.test_group }}
@@ -641,13 +719,13 @@ jobs:
- name: Download Cloudberry RPM
uses: actions/download-artifact@v4
with:
- name: cloudberry-rpm
+ name: cloudberry-rpm-cb2
path: /tmp
- name: Download Cloudberry source (Rocky 9)
uses: actions/download-artifact@v4
with:
- name: cloudberry-source-rocky9
+ name: cloudberry-source-rocky9-cb2
path: /tmp
- name: Download singlecluster Rocky 9 image
@@ -662,7 +740,7 @@ jobs:
- name: Prepare Cloudberry source
run: |
- tar xzf /tmp/cloudberry-source-rocky9.tar.gz
+ tar xzf /tmp/cloudberry-source-rocky9-cb2.tar.gz
chmod -R u+rwX,go+rX cloudberry
- name: Restore Maven cache
@@ -688,7 +766,10 @@ jobs:
docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin
/home/gpadmin/workspace/cloudberry
docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin
/home/gpadmin/.m2
docker cp /tmp/*.rpm pxf-cbdb-dev:/tmp/
- docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm
+ docker cp /tmp/cloudberry-build.ref pxf-cbdb-dev:/tmp/
+ docker cp /tmp/cloudberry-build.sha pxf-cbdb-dev:/tmp/
+ docker cp /tmp/cloudberry-build.pg-major pxf-cbdb-dev:/tmp/
+ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm
/tmp/cloudberry-build.*
docker exec pxf-cbdb-dev bash -lc "cd
/home/gpadmin/workspace/cloudberry-pxf &&
./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh"
- name: Run Test - ${{ matrix.test_group }}
@@ -1156,7 +1237,7 @@ jobs:
# Stage 3: Summary job
test-summary:
name: Test Summary
- needs: [pxf-test, pxf-test-rocky9, pxf-test-rocky10,
pxf-testcontainer-test]
+ needs: [native-extension-compatibility, pxf-test, pxf-test-rocky9,
pxf-test-rocky10, pxf-testcontainer-test]
if: always()
runs-on: ubuntu-latest
steps:
@@ -1249,6 +1330,11 @@ jobs:
EXIT_CODE=0
+ if [ "${{ needs['native-extension-compatibility'].result }}" !=
"success" ]; then
+ echo "::error::Cloudberry 2.x/3.x native extension compatibility
check failed"
+ EXIT_CODE=1
+ fi
+
# Ubuntu results (test-results-<group>, excluding
test-results-rocky*-*)
generate_os_summary "Ubuntu 22.04" "test-results-[!r]*" || EXIT_CODE=1
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
index 350cc91f..5833234c 100755
--- a/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
+++ b/automation/src/main/resources/testcontainers/pxf-cbdb/script/build_pxf.sh
@@ -19,6 +19,7 @@
#
# --------------------------------------------------------------------
# Build and install PXF — works on both Ubuntu and Rocky/RHEL
+set -euo pipefail
# Auto-detect Java 11 path
if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ]; then
@@ -45,8 +46,13 @@ sudo chmod -R a+rwX "$PXF_HOME"
# Build and Install PXF
cd /home/gpadmin/workspace/cloudberry-pxf
-make -C external-table install
-make -C fdw install
+# external-table and fdw are native extensions; the automation harness copies
+# the host repo (including any binaries the host already built) into this
+# container, so a plain `make install` would silently reuse a .so linked
+# against the host's glibc instead of rebuilding for this container's glibc.
+# Force a clean rebuild here.
+make -C external-table clean install
+make -C fdw clean install
make -C server install-server
make -C server install-jdbc-drivers
diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
index cf99df59..2e444da7 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh
@@ -198,17 +198,59 @@ install_build_deps() {
fi
}
+verify_cloudberry_install() {
+ local actual_version actual_pg_major
+ actual_version=$(/usr/local/cloudberry-db/bin/pg_config --version)
+ actual_pg_major=${actual_version#* }
+ actual_pg_major=${actual_pg_major%%.*}
+
+ log "installed Cloudberry reports ${actual_version}"
+ /usr/local/cloudberry-db/bin/postgres --gp-version
+
+ local ref_file=/tmp/cloudberry-build.ref
+ local sha_file=/tmp/cloudberry-build.sha
+ local pg_major_file=/tmp/cloudberry-build.pg-major
+ if [ ! -e "$ref_file" ] && [ ! -e "$sha_file" ] && [ ! -e "$pg_major_file"
]; then
+ log "Cloudberry build identity is unavailable; skipping expected-version
check"
+ return
+ fi
+
+ [ -r "$ref_file" ] || die "Missing Cloudberry build identity: $ref_file"
+ [ -r "$sha_file" ] || die "Missing Cloudberry build identity: $sha_file"
+ [ -r "$pg_major_file" ] || die "Missing Cloudberry build identity:
$pg_major_file"
+
+ local expected_ref expected_sha expected_pg_major actual_sha
+ expected_ref=$(<"$ref_file")
+ expected_sha=$(<"$sha_file")
+ expected_pg_major=$(<"$pg_major_file")
+ [ "$actual_pg_major" = "$expected_pg_major" ] ||
+ die "Expected PostgreSQL ${expected_pg_major} from ${expected_ref}, got
${actual_version}"
+
+ if git -C /home/gpadmin/workspace/cloudberry rev-parse HEAD >/dev/null 2>&1;
then
+ actual_sha=$(git -C /home/gpadmin/workspace/cloudberry rev-parse HEAD)
+ [ "$actual_sha" = "$expected_sha" ] ||
+ die "Cloudberry package/source mismatch: expected ${expected_sha}, got
${actual_sha}"
+ fi
+ log "verified Cloudberry ref=${expected_ref} commit=${expected_sha}
pg_major=${expected_pg_major}"
+}
+
install_cloudberry_from_package() {
log "installing Cloudberry from package"
- local pkg_file=""
+ local package_pattern
if [ "$OS_FAMILY" = "deb" ]; then
- pkg_file=$(find /tmp -name "apache-cloudberry-db*.deb" 2>/dev/null | head
-1)
- [ -z "$pkg_file" ] && die "No .deb package found in /tmp"
+ package_pattern="apache-cloudberry-db*.deb"
else
- pkg_file=$(find /tmp -name "apache-cloudberry-db*.rpm" 2>/dev/null | head
-1)
- [ -z "$pkg_file" ] && die "No .rpm package found in /tmp"
+ package_pattern="apache-cloudberry-db*.rpm"
+ fi
+
+ local -a package_files=()
+ mapfile -t package_files < <(find /tmp -maxdepth 1 -type f -name
"$package_pattern" -print)
+ if [ "${#package_files[@]}" -ne 1 ]; then
+ find /tmp -maxdepth 1 -type f -name "$package_pattern" -print || true
+ die "Expected exactly one Cloudberry package matching ${package_pattern},
found ${#package_files[@]}"
fi
+ local pkg_file=${package_files[0]}
install_build_deps
@@ -254,6 +296,7 @@ EOF
# Initialize and start Cloudberry cluster
source /usr/local/cloudberry-db/cloudberry-env.sh
+ verify_cloudberry_install
make create-demo-cluster -C ~/workspace/cloudberry || {
log "create-demo-cluster failed, trying manual setup"
cd ~/workspace/cloudberry
diff --git a/external-table/src/gpdbwritableformatter.c
b/external-table/src/gpdbwritableformatter.c
index 986da281..c8bececb 100644
--- a/external-table/src/gpdbwritableformatter.c
+++ b/external-table/src/gpdbwritableformatter.c
@@ -45,6 +45,7 @@
#include "utils/syscache.h"
#include "utils/lsyscache.h"
+#include <arpa/inet.h>
#include <unistd.h>
#include "access/external.h"
diff --git a/external-table/src/pxfuriparser.c
b/external-table/src/pxfuriparser.c
index 47fce006..0bcdba2b 100644
--- a/external-table/src/pxfuriparser.c
+++ b/external-table/src/pxfuriparser.c
@@ -291,7 +291,7 @@ GPHDUri_verify_no_duplicate_options(GPHDUri *uri)
{
OptionData *data = (OptionData *) lfirst(option);
- Value *key = makeString(asc_toupper(data->key,
strlen(data->key)));
+ Node *key = (Node *) makeString(asc_toupper(data->key,
strlen(data->key)));
if (!list_member(previousKeys, key))
previousKeys = lappend(previousKeys, key);
@@ -308,7 +308,7 @@ GPHDUri_verify_no_duplicate_options(GPHDUri *uri)
initStringInfo(&duplicates);
foreach(key, duplicateKeys)
{
- char *keyname = strVal((Value *) lfirst(key));
+ char *keyname = strVal((Node *) lfirst(key));
if (!first)
appendStringInfoString(&duplicates, ", ");
diff --git a/external-table/test/pxfprotocol_test.c
b/external-table/test/pxfprotocol_test.c
index f19e6aeb..c7d5eb77 100644
--- a/external-table/test/pxfprotocol_test.c
+++ b/external-table/test/pxfprotocol_test.c
@@ -47,7 +47,7 @@ test_pxfprotocol_validate_urls(void **state)
PG_FUNCTION_ARGS = palloc0(sizeof(FunctionCallInfoData));
fcinfo->context = palloc0(sizeof(ExtProtocolValidatorData));
fcinfo->context->type = T_ExtProtocolValidatorData;
- Value *v = makeString(uri_no_profile);
+ Node *v = (Node *) makeString(uri_no_profile);
List *list = list_make1(v);
((ExtProtocolValidatorData *) fcinfo->context)->url_list = list;
diff --git a/fdw/pxf_option.c b/fdw/pxf_option.c
index 13ccfe92..a89e71e9 100644
--- a/fdw/pxf_option.c
+++ b/fdw/pxf_option.c
@@ -470,7 +470,7 @@ PxfGetOptions(Oid foreigntableid)
copy_options = lappend(copy_options, def);
else
{
- Value *val = makeString(def->defname);
+ Node *val = (Node *) makeString(def->defname);
/*
* if we have already seen this option before disregard
the new
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
