This is an automated email from the ASF dual-hosted git repository.
ostinru pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 1c7b8529 PXF3: Prepare to Java 17 - split HBase push down into Java 8
library (#162)
1c7b8529 is described below
commit 1c7b8529c68982a64ce9e027c1a4ab9418cadfd2
Author: Nikolay Antonov <[email protected]>
AuthorDate: Fri Oct 2 17:57:38 2026 +0500
PXF3: Prepare to Java 17 - split HBase push down into Java 8 library (#162)
Adapt GreengageDB/pxf@edf1aaffa5ff5133feea49a4a9c4a001c5836abd to
Cloudberry package names and HBase 2.5.15.
The three numeric filter comparators are packaged in a dedicated Java
8-compatible `pxf-hbase-lib` JAR, staged under share, and copied to the HBase
RegionServer lib directory by the automation setup.
---
LICENSE | 19 ++++++++++++++++++
automation/Makefile | 2 +-
automation/pom.xml | 8 ++++++++
ci/docker/pxf-cbdb-dev/common/script/run_tests.sh | 12 ++++++-----
docs/access-hadoop/hbase_pxf.md | 2 +-
pom.xml | 19 ++++++++++++++++++
server/build.gradle | 6 +++---
server/pxf-hbase-lib/build.gradle | 23 ++++++++++++++++++++++
.../hbase/utilities/HBaseDoubleComparator.java | 0
.../hbase/utilities/HBaseFloatComparator.java | 0
.../hbase/utilities/HBaseIntegerComparator.java | 1 +
server/pxf-hbase/build.gradle | 1 +
server/settings.gradle | 1 +
13 files changed, 84 insertions(+), 10 deletions(-)
diff --git a/LICENSE b/LICENSE
index a95bb07c..78ba0ced 100644
--- a/LICENSE
+++ b/LICENSE
@@ -267,3 +267,22 @@ Apache Cloudberry PXF includes codes from
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+
+----------------------------
+ Apache License - Version 2.0
+
+ server/pxf-hbase-lib/build.gradle
+
+ Copyright (C) 2024 Arenadata
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/automation/Makefile b/automation/Makefile
index 137f6d82..ca3d64b7 100755
--- a/automation/Makefile
+++ b/automation/Makefile
@@ -105,7 +105,7 @@ symlink_pxf_jars: check-env
unzip -qq -j "$${pxf_app}" 'BOOT-INF/lib/pxf-*.jar' -d
$(PXF_TMP_LIB)/tmp && \
for jar in $(PXF_TMP_LIB)/tmp/pxf-*.jar; do \
jar_name="$${jar##*/}"; \
- if [[ $${jar_name} =~
^pxf-[A-Za-z0-9]+(-[0-9.]+.*).jar$$ ]]; then \
+ if [[ $${jar_name} =~
^pxf-[A-Za-z0-9-]+(-[0-9.]+.*).jar$$ ]]; then \
link=$(PXF_TMP_LIB)/$${jar_name/$${BASH_REMATCH[1]}/}; \
echo "copying $${link} -> $${jar}"; \
cp "$${jar}" "$${link}"; \
diff --git a/automation/pom.xml b/automation/pom.xml
index e318ec6a..3b79ffb5 100644
--- a/automation/pom.xml
+++ b/automation/pom.xml
@@ -108,6 +108,14 @@
<version>LATEST</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.cloudberry</groupId>
+ <artifactId>pxf-hbase-lib</artifactId>
+ <scope>system</scope>
+ <systemPath>${pxf.lib}/pxf-hbase-lib.jar</systemPath>
+ <version>LATEST</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.cloudberry</groupId>
<artifactId>pxf-hdfs</artifactId>
diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
index 89690031..53063d29 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
@@ -111,12 +111,14 @@ cleanup_hive_state() {
}
start_hbase() {
- echo "[run_tests] copying pxf-hbase.jar to HBase lib..."
- cp /home/gpadmin/automation_tmp_lib/pxf-hbase.jar "${GPHD_ROOT}/hbase/lib/"
2>/dev/null || true
- if [ ! -f "${GPHD_ROOT}/hbase/lib/pxf-hbase.jar" ]; then
- pxf_app=$(ls -1v /usr/local/pxf/application/pxf-app-*.jar 2>/dev/null |
grep -v 'plain' | tail -n 1)
- [ -n "${pxf_app}" ] && unzip -qq -j "${pxf_app}"
'BOOT-INF/lib/pxf-hbase-*.jar' -d "${GPHD_ROOT}/hbase/lib/" || true
+ echo "[run_tests] copying Java 8 HBase comparator library to HBase lib..."
+ local comparator_jar
+ comparator_jar=$(find "${PXF_HOME}/share" -maxdepth 1 -name
'pxf-hbase-lib-*.jar' -print -quit)
+ if [[ -z "${comparator_jar}" ]]; then
+ echo "[run_tests] ERROR: HBase comparator library not found in
${PXF_HOME}/share"
+ return 1
fi
+ cp "${comparator_jar}" "${GPHD_ROOT}/hbase/lib/pxf-hbase-lib.jar"
if pgrep -f HMaster >/dev/null 2>&1; then
echo "[run_tests] HBase HMaster already running, skipping start"
else
diff --git a/docs/access-hadoop/hbase_pxf.md b/docs/access-hadoop/hbase_pxf.md
index e3332f33..fae08c59 100644
--- a/docs/access-hadoop/hbase_pxf.md
+++ b/docs/access-hadoop/hbase_pxf.md
@@ -14,7 +14,7 @@ This section describes how to use the PXF HBase connector.
Before working with HBase table data, ensure that you have:
-- Copied `<PXF_INSTALL_DIR>/share/pxf-hbase-*.jar` to each node in your HBase
cluster, and that the location of this PXF JAR file is in the
`$HBASE_CLASSPATH`. This configuration is required for the PXF HBase connector
to support filter pushdown.
+- Copied `<PXF_INSTALL_DIR>/share/pxf-hbase-lib-*.jar` to each node in your
HBase cluster, and added the JAR to `$HBASE_CLASSPATH`. This JAR contains the
filter comparators used by HBase RegionServers and is compatible with Java 8.
Remove any older full `pxf-hbase` JAR from the RegionServer classpath.
- Met the PXF Hadoop [Prerequisites](./access_hdfs.md#prerequisites).
## HBase Primer
diff --git a/pom.xml b/pom.xml
index b0ecbb05..d5c111b6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1327,6 +1327,25 @@
<!-- End of the legacy Greenplum PXF files -->
+ <!--
+ The following files are part of the Greengage PXF
+ open-source files, licensed under the Apache
+ License 2.0 by Greenplum/Pivotal/VMware.
+
+ Original files located:
+ https://github.com/GreengageDB/pxf
+
+ These files do not contain license headers or recognizable
+ license markers, but are Apache-licensed by virtue of their
+ inclusion in the Greengage project. We exclude them
+ from Apache RAT scans to avoid false positives, while
+ retaining them in the source distribution.
+ -->
+ <exclude>server/pxf-hbase-lib/build.gradle</exclude>
+
+ <!-- End of the Greengage files -->
+
+
<!-- The following files are introduced by Cloudberry,
but need to be excluded from the RAT scan.
-->
diff --git a/server/build.gradle b/server/build.gradle
index 1f7af772..9ce5039b 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -358,10 +358,10 @@ private static void printResults(result) {
task stage(type: Copy) {
description "Build PXF server and stage resulting artifacts for packaging."
dependsOn ":pxf-service:bootJar"
+ dependsOn ":pxf-hbase-lib:jar"
- // Provide the pxf-hbase jar. The pxf-hbase.jar is required for the PXF
- // HBase connector to support filter push down
- from("pxf-hbase/build/libs") { into 'share' }
+ // RegionServers need these Java 8-compatible comparators for numeric
filter pushdown.
+ from("pxf-hbase-lib/build/libs") { into 'share' }
from("pxf-service/build/libs") {
into 'application'
rename 'pxf-service-(.+)', 'pxf-app-$1'
diff --git a/server/pxf-hbase-lib/build.gradle
b/server/pxf-hbase-lib/build.gradle
new file mode 100644
index 00000000..1dd452bf
--- /dev/null
+++ b/server/pxf-hbase-lib/build.gradle
@@ -0,0 +1,23 @@
+apply plugin: "java-library"
+
+jar {
+ enabled = true
+}
+
+compileJava {
+ options.release = 8
+}
+
+dependencies {
+ implementation("org.apache.hbase:hbase-client") {
transitive = false }
+ implementation("org.apache.hbase:hbase-common") {
transitive = false }
+ implementation("org.apache.hbase:hbase-protocol-shaded") {
transitive = false }
+ implementation("org.apache.hbase.thirdparty:hbase-shaded-protobuf") {
transitive = false }
+ implementation("org.apache.yetus:audience-annotations") {
transitive = false }
+
+ compileOnly("com.google.code.findbugs:annotations")
+}
+
+test {
+ useJUnitPlatform()
+}
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
b/server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
similarity index 100%
rename from
server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
rename to
server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseDoubleComparator.java
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
b/server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
similarity index 100%
rename from
server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
rename to
server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseFloatComparator.java
diff --git
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
b/server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
similarity index 97%
rename from
server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
rename to
server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
index a1e589c5..7fa04d96 100644
---
a/server/pxf-hbase/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
+++
b/server/pxf-hbase-lib/src/main/java/org/apache/cloudberry/pxf/plugins/hbase/utilities/HBaseIntegerComparator.java
@@ -41,6 +41,7 @@ import
org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferExce
* ({@link #toByteArray()} and {@link #parseFrom(byte[])} methods).
* <p>
* A reference can be found in {@link SubstringComparator}.
+ * This class MUST ONLY use features from java 8 and lower as it will be
loaded within hbase
*/
public class HBaseIntegerComparator extends ByteArrayComparable {
diff --git a/server/pxf-hbase/build.gradle b/server/pxf-hbase/build.gradle
index 6d69f3ef..126332b8 100644
--- a/server/pxf-hbase/build.gradle
+++ b/server/pxf-hbase/build.gradle
@@ -20,6 +20,7 @@ dependencies {
*******************************/
implementation(project(':pxf-api'))
+ implementation(project(':pxf-hbase-lib'))
implementation("commons-collections:commons-collections")
/*******************************
diff --git a/server/settings.gradle b/server/settings.gradle
index 5df3ff21..cd1d634f 100644
--- a/server/settings.gradle
+++ b/server/settings.gradle
@@ -21,6 +21,7 @@ rootProject.name = 'pxf'
include 'pxf-service',
'pxf-api',
+ 'pxf-hbase-lib',
'pxf-hbase',
'pxf-hdfs',
'pxf-hive',
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
