This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 326af03c [python] Discover an optionally packaged OSS C++ bridge
(#1043)
326af03c is described below
commit 326af03cff7b1859d162aff79036d646a0475a16
Author: XiaoHongbo <[email protected]>
AuthorDate: Wed Oct 7 10:12:26 2026 +0800
[python] Discover an optionally packaged OSS C++ bridge (#1043)
---
bindings/python/src/context.rs | 9 ++-
bindings/python/src/lib.rs | 1 +
bindings/python/src/oss_cpp.rs | 153 +++++++++++++++++++++++++++++++++++++++++
bindings/python/src/table.rs | 6 +-
integrations/oss-cpp/README.md | 5 ++
5 files changed, 168 insertions(+), 6 deletions(-)
diff --git a/bindings/python/src/context.rs b/bindings/python/src/context.rs
index b661f14b..3bb58b04 100644
--- a/bindings/python/src/context.rs
+++ b/bindings/python/src/context.rs
@@ -140,7 +140,10 @@ fn complete_jindo_options_with(
options
}
-pub(crate) fn complete_jindo_options(options: HashMap<String, String>) ->
HashMap<String, String> {
+pub(crate) fn complete_storage_options(
+ options: HashMap<String, String>,
+) -> HashMap<String, String> {
+ let options = crate::oss_cpp::complete_options(options);
let library_path = std::env::var_os(JINDOSDK_LIBRARY_PATH);
let home = std::env::var_os(JINDOSDK_HOME);
complete_jindo_options_with(
@@ -152,7 +155,7 @@ pub(crate) fn complete_jindo_options(options:
HashMap<String, String>) -> HashMa
}
fn build_paimon_catalog(catalog_options: HashMap<String, String>) ->
PyResult<Arc<dyn Catalog>> {
- let catalog_options = complete_jindo_options(catalog_options);
+ let catalog_options = complete_storage_options(catalog_options);
let rt = runtime();
rt.block_on(async {
let options = Options::from_map(catalog_options);
@@ -584,7 +587,7 @@ impl PySQLContext {
catalog_options: HashMap<String, String>,
default_database: Option<String>,
) -> PyResult<()> {
- let catalog_options = complete_jindo_options(catalog_options);
+ let catalog_options = complete_storage_options(catalog_options);
let rt = runtime();
py.detach(move || {
rt.block_on(async {
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 6756fc7a..a4c8eaa1 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -20,6 +20,7 @@ use pyo3::prelude::*;
mod blob;
mod context;
mod error;
+mod oss_cpp;
mod predicate;
mod read;
mod schema;
diff --git a/bindings/python/src/oss_cpp.rs b/bindings/python/src/oss_cpp.rs
new file mode 100644
index 00000000..ad2b4314
--- /dev/null
+++ b/bindings/python/src/oss_cpp.rs
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use pyo3::prelude::*;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+const LIBRARY_PATH: &str = "fs.oss.cpp.library.path";
+
+pub(crate) fn complete_options(options: HashMap<String, String>) ->
HashMap<String, String> {
+ complete_options_with(options, || discover_library("pypaimon_oss_cpp"))
+}
+
+fn complete_options_with(
+ mut options: HashMap<String, String>,
+ discover: impl FnOnce() -> Option<PathBuf>,
+) -> HashMap<String, String> {
+ if options
+ .get("fs.oss.impl")
+ .is_some_and(|v| v.eq_ignore_ascii_case("cpp"))
+ && !options.contains_key(LIBRARY_PATH)
+ {
+ if let Some(path) = discover() {
+ if let Some(path) = path.to_str() {
+ options.insert(LIBRARY_PATH.into(), path.into());
+ }
+ }
+ }
+ options
+}
+
+fn discover_library(package: &str) -> Option<PathBuf> {
+ Python::attach(|py| {
+ // Inspect a top-level package without importing its initialization
code.
+ let spec = py
+ .import("importlib.util")
+ .ok()?
+ .call_method1("find_spec", (package,))
+ .ok()?;
+ let origin = spec.getattr("origin").ok()?.extract::<String>().ok()?;
+ library_in(Path::new(&origin).parent()?)
+ })
+}
+
+fn library_in(directory: &Path) -> Option<PathBuf> {
+ let name = if cfg!(target_os = "macos") {
+ "liboss_cpp_bridge.dylib"
+ } else if cfg!(target_os = "linux") {
+ "liboss_cpp_bridge.so"
+ } else {
+ return None;
+ };
+ let path = directory.join(name);
+ path.is_file().then_some(path)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn fills_only_missing_cpp_path() {
+ for backend in ["cpp", "CPP", "Cpp", "cPp"] {
+ let options = HashMap::from([("fs.oss.impl".into(),
backend.into())]);
+ let result =
+ complete_options_with(options.clone(), ||
Some("/package/bridge.so".into()));
+ assert_eq!(result[LIBRARY_PATH], "/package/bridge.so");
+ assert_eq!(result["fs.oss.impl"], backend);
+ assert_eq!(complete_options_with(options.clone(), || None),
options);
+ }
+ }
+
+ #[test]
+ fn explicit_paths_are_never_replaced() {
+ for path in ["/missing/explicit.so", ""] {
+ let options = HashMap::from([
+ ("fs.oss.impl".into(), "cpp".into()),
+ (LIBRARY_PATH.into(), path.into()),
+ ]);
+ assert_eq!(
+ complete_options_with(options.clone(), || panic!("must not
discover")),
+ options
+ );
+ }
+ }
+
+ #[test]
+ fn other_backends_do_not_discover() {
+ for backend in [None, Some("jindo"), Some("legacy"), Some(" cpp ")] {
+ let options = backend
+ .map(|v| HashMap::from([("fs.oss.impl".into(), v.into())]))
+ .unwrap_or_default();
+ assert_eq!(
+ complete_options_with(options.clone(), || panic!("must not
discover")),
+ options
+ );
+ }
+ }
+
+ #[test]
+ fn finds_only_the_platform_library() {
+ let directory =
+ std::env::temp_dir().join(format!("paimon-oss-cpp-discovery-{}",
std::process::id()));
+ std::fs::create_dir_all(&directory).unwrap();
+ assert!(library_in(&directory).is_none());
+ let name = if cfg!(target_os = "macos") {
+ "liboss_cpp_bridge.dylib"
+ } else {
+ "liboss_cpp_bridge.so"
+ };
+ let path = directory.join(name);
+ std::fs::write(&path, []).unwrap();
+ if cfg!(any(target_os = "macos", target_os = "linux")) {
+ assert_eq!(library_in(&directory), Some(path.clone()));
+ Python::attach(|py| {
+ let name = "_paimon_oss_cpp_discovery_test";
+ let module = PyModule::new(py, name).unwrap();
+ let spec = py
+ .import("importlib.machinery")
+ .unwrap()
+ .getattr("ModuleSpec")
+ .unwrap()
+ .call1((name, py.None()))
+ .unwrap();
+ spec.setattr("origin",
directory.join("__init__.py").to_str().unwrap())
+ .unwrap();
+ module.setattr("__spec__", spec).unwrap();
+ let modules =
py.import("sys").unwrap().getattr("modules").unwrap();
+ modules.set_item(name, module).unwrap();
+ let result = discover_library(name);
+ modules.del_item(name).unwrap();
+ assert_eq!(result, Some(path.clone()));
+
assert!(discover_library("_paimon_nonexistent_oss_cpp_package").is_none());
+ });
+ }
+ std::fs::remove_file(path).unwrap();
+ std::fs::remove_dir(directory).unwrap();
+ }
+}
diff --git a/bindings/python/src/table.rs b/bindings/python/src/table.rs
index 95e1d82f..ed19d070 100644
--- a/bindings/python/src/table.rs
+++ b/bindings/python/src/table.rs
@@ -27,7 +27,7 @@ use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::PyDict;
-use crate::context::complete_jindo_options;
+use crate::context::complete_storage_options;
use crate::error::to_py_err;
use crate::partition::PyPartitionStat;
use crate::read::PyReadBuilder;
@@ -67,7 +67,7 @@ impl PyTable {
) -> PyResult<Self> {
let schema: TableSchema = serde_json::from_str(schema_json)
.map_err(|err| PyValueError::new_err(format!("Invalid table schema
JSON: {err}")))?;
- let properties = complete_jindo_options(
+ let properties = complete_storage_options(
options
.map(crate::read::extract_options)
.transpose()?
@@ -103,7 +103,7 @@ impl PyTable {
PyValueError::new_err(format!("Invalid REST table response
JSON: {err}"))
})?;
let identifier = Identifier::new(database, table);
- let rest_options = complete_jindo_options(rest_options);
+ let rest_options = complete_storage_options(rest_options);
let table = py
.detach(|| {
runtime().block_on(paimon::table::Table::from_rest_response(
diff --git a/integrations/oss-cpp/README.md b/integrations/oss-cpp/README.md
index 77adf236..0349a472 100644
--- a/integrations/oss-cpp/README.md
+++ b/integrations/oss-cpp/README.md
@@ -52,6 +52,11 @@ Use the existing `fs.oss.accessKeyId`,
`fs.oss.accessKeySecret` and optional
`fs.oss.securityToken` credentials. REST catalog credentials still refresh;
static FileIO credentials do not.
+Python bindings also discover `liboss_cpp_bridge.so` (Linux) or
+`liboss_cpp_bridge.dylib` (macOS) beside `pypaimon_oss_cpp/__init__.py`.
+An explicit `fs.oss.cpp.library.path` takes precedence. The companion package
+is not yet published.
+
| Option | Default |
|---|---:|
| `fs.oss.cpp.max.concurrent.requests` | 8 |
