This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 066a80300 feat(bindings/java): support presign ops (#3069)
066a80300 is described below
commit 066a803009c0dc854f7eac885d8b6a7bb2d60f78
Author: tison <[email protected]>
AuthorDate: Thu Sep 14 23:02:01 2023 +0800
feat(bindings/java): support presign ops (#3069)
* feat: hashmap_to_jmap
Signed-off-by: tison <[email protected]>
* feat: make_presigned_request
Signed-off-by: tison <[email protected]>
* feat: bind presign methods
Signed-off-by: tison <[email protected]>
* implement behavior tests
Signed-off-by: tison <[email protected]>
* tidy tests
Signed-off-by: tison <[email protected]>
* better utilities
Signed-off-by: tison <[email protected]>
---------
Signed-off-by: tison <[email protected]>
---
bindings/java/README.md | 13 +-
bindings/java/src/error.rs | 18 +--
bindings/java/src/lib.rs | 40 +++++-
.../src/main/java/org/apache/opendal/Operator.java | 22 ++++
.../java/org/apache/opendal/PresignedRequest.java | 41 ++++++
bindings/java/src/operator.rs | 145 ++++++++++++++++++++-
.../java/org/apache/opendal/AsyncStepsTest.java | 13 +-
.../test/java/org/apache/opendal/OperatorTest.java | 14 +-
.../java/org/apache/opendal/RedisServiceTest.java | 14 +-
.../condition/OpenDALExceptionCondition.java | 70 ++++++++++
10 files changed, 355 insertions(+), 35 deletions(-)
diff --git a/bindings/java/README.md b/bindings/java/README.md
index 3a2edacfe..6523b00d4 100644
--- a/bindings/java/README.md
+++ b/bindings/java/README.md
@@ -59,7 +59,7 @@ This project provides OpenDAL Java bindings with artifact
name `opendal-java`. I
You can use Maven to build both Rust dynamic lib and JAR files with one
command now:
```shell
-mvn clean package -DskipTests=true
+./mvnw clean package -DskipTests=true
```
## Run tests
@@ -69,13 +69,20 @@ Currently, all tests are written in Java. It contains the
Cucumber feature tests
You can run tests with the following command:
```shell
-mvn clean verify
+./mvnw clean verify -Dcargo-build.features=services-redis
```
+> **Note:**
+>
+> The `-Dcargo-build.features=services-redis` argument is a temporary
workaround. See also:
+>
+> * https://github.com/apache/incubator-opendal/pull/3060
+> * https://github.com/apache/incubator-opendal/issues/3066
+
Additionally, this project uses
[spotless](https://github.com/diffplug/spotless) for code formatting so that
all developers share a consistent code style without bikeshedding on it.
You can apply the code style with the following command::
```shell
-mvn spotless:apply
+./mvnw spotless:apply
```
diff --git a/bindings/java/src/error.rs b/bindings/java/src/error.rs
index 8a99c4f49..2d13d6754 100644
--- a/bindings/java/src/error.rs
+++ b/bindings/java/src/error.rs
@@ -29,6 +29,12 @@ pub(crate) struct Error {
}
impl Error {
+ pub(crate) fn unexpected(err: impl Into<anyhow::Error> + Display) -> Error
{
+ Error {
+ inner: opendal::Error::new(ErrorKind::Unexpected,
&err.to_string()).set_source(err),
+ }
+ }
+
pub(crate) fn throw(&self, env: &mut JNIEnv) {
if let Err(err) = self.do_throw(env) {
match err {
@@ -84,25 +90,19 @@ impl From<opendal::Error> for Error {
impl From<jni::errors::Error> for Error {
fn from(error: jni::errors::Error) -> Self {
- Self {
- inner: opendal::Error::new(ErrorKind::Unexpected,
&error.to_string()).set_source(error),
- }
+ Error::unexpected(error)
}
}
impl From<std::str::Utf8Error> for Error {
fn from(error: std::str::Utf8Error) -> Self {
- Self {
- inner: opendal::Error::new(ErrorKind::Unexpected,
&error.to_string()).set_source(error),
- }
+ Error::unexpected(error)
}
}
impl From<std::string::FromUtf8Error> for Error {
fn from(error: std::string::FromUtf8Error) -> Self {
- Self {
- inner: opendal::Error::new(ErrorKind::Unexpected,
&error.to_string()).set_source(error),
- }
+ Error::unexpected(error)
}
}
diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs
index f838cff56..e1703b527 100644
--- a/bindings/java/src/lib.rs
+++ b/bindings/java/src/lib.rs
@@ -19,14 +19,16 @@ use std::cell::RefCell;
use std::collections::HashMap;
use std::ffi::c_void;
-use jni::objects::JMap;
+use crate::error::Error;
use jni::objects::JObject;
use jni::objects::JString;
+use jni::objects::{JMap, JValue};
use jni::sys::jint;
use jni::sys::JNI_VERSION_1_8;
use jni::JNIEnv;
use jni::JavaVM;
use once_cell::sync::OnceCell;
+use opendal::raw::PresignedRequest;
use tokio::runtime::Builder;
use tokio::runtime::Runtime;
@@ -104,3 +106,39 @@ fn jmap_to_hashmap(env: &mut JNIEnv, params: &JObject) ->
Result<HashMap<String,
Ok(result)
}
+
+fn hashmap_to_jmap<'a>(env: &mut JNIEnv<'a>, map: &HashMap<String, String>) ->
Result<JObject<'a>> {
+ let map_object = env.new_object("java/util/HashMap", "()V", &[])?;
+ let jmap = env.get_map(&map_object)?;
+ for (k, v) in map {
+ let key = env.new_string(k)?;
+ let value = env.new_string(v)?;
+ jmap.put(env, &key, &value)?;
+ }
+ Ok(map_object)
+}
+
+fn make_presigned_request<'a>(env: &mut JNIEnv<'a>, req: PresignedRequest) ->
Result<JObject<'a>> {
+ let method = env.new_string(req.method().as_str())?;
+ let uri = env.new_string(req.uri().to_string())?;
+ let headers = {
+ let mut map = HashMap::new();
+ for (k, v) in req.header().iter() {
+ let key = k.to_string();
+ let value = v.to_str().map_err(Error::unexpected)?;
+ map.insert(key, value.to_owned());
+ }
+ map
+ };
+ let headers = hashmap_to_jmap(env, &headers)?;
+ let result = env.new_object(
+ "org/apache/opendal/PresignedRequest",
+ "(Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;)V",
+ &[
+ JValue::Object(&method),
+ JValue::Object(&uri),
+ JValue::Object(&headers),
+ ],
+ )?;
+ Ok(result)
+}
diff --git a/bindings/java/src/main/java/org/apache/opendal/Operator.java
b/bindings/java/src/main/java/org/apache/opendal/Operator.java
index 650c6ab2a..4cd6c2c15 100644
--- a/bindings/java/src/main/java/org/apache/opendal/Operator.java
+++ b/bindings/java/src/main/java/org/apache/opendal/Operator.java
@@ -20,6 +20,7 @@
package org.apache.opendal;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@@ -141,6 +142,21 @@ public class Operator extends NativeObject {
return AsyncRegistry.take(requestId);
}
+ public CompletableFuture<Void> presignRead(String path, Duration duration)
{
+ final long requestId = presignRead(nativeHandle, path,
duration.toNanos());
+ return AsyncRegistry.take(requestId);
+ }
+
+ public CompletableFuture<Void> presignWrite(String path, Duration
duration) {
+ final long requestId = presignWrite(nativeHandle, path,
duration.toNanos());
+ return AsyncRegistry.take(requestId);
+ }
+
+ public CompletableFuture<Void> presignStat(String path, Duration duration)
{
+ final long requestId = presignStat(nativeHandle, path,
duration.toNanos());
+ return AsyncRegistry.take(requestId);
+ }
+
public CompletableFuture<Void> delete(String path) {
final long requestId = delete(nativeHandle, path);
return AsyncRegistry.take(requestId);
@@ -160,4 +176,10 @@ public class Operator extends NativeObject {
private static native long delete(long nativeHandle, String path);
private static native long stat(long nativeHandle, String path);
+
+ private static native long presignRead(long nativeHandle, String path,
long duration);
+
+ private static native long presignWrite(long nativeHandle, String path,
long duration);
+
+ private static native long presignStat(long nativeHandle, String path,
long duration);
}
diff --git
a/bindings/java/src/main/java/org/apache/opendal/PresignedRequest.java
b/bindings/java/src/main/java/org/apache/opendal/PresignedRequest.java
new file mode 100644
index 000000000..c14f9faf8
--- /dev/null
+++ b/bindings/java/src/main/java/org/apache/opendal/PresignedRequest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.opendal;
+
+import java.util.Map;
+import lombok.Data;
+
+@Data
+public class PresignedRequest {
+ /**
+ * HTTP method of this request.
+ */
+ private final String method;
+
+ /**
+ * URI of this request.
+ */
+ private final String uri;
+
+ /**
+ * HTTP headers of this request.
+ */
+ private final Map<String, String> headers;
+}
diff --git a/bindings/java/src/operator.rs b/bindings/java/src/operator.rs
index b06b09067..f18a3cb0a 100644
--- a/bindings/java/src/operator.rs
+++ b/bindings/java/src/operator.rs
@@ -16,6 +16,7 @@
// under the License.
use std::str::FromStr;
+use std::time::Duration;
use jni::objects::JByteArray;
use jni::objects::JClass;
@@ -26,13 +27,14 @@ use jni::objects::JValueOwned;
use jni::sys::jlong;
use jni::JNIEnv;
use opendal::layers::BlockingLayer;
+use opendal::raw::PresignedRequest;
use opendal::Operator;
use opendal::Scheme;
-use crate::get_current_env;
use crate::get_global_runtime;
use crate::jmap_to_hashmap;
use crate::Result;
+use crate::{get_current_env, make_presigned_request};
#[no_mangle]
pub extern "system" fn Java_org_apache_opendal_Operator_constructor(
@@ -260,6 +262,147 @@ async fn do_delete(op: &mut Operator, path: String) ->
Result<()> {
Ok(op.delete(&path).await?)
}
+/// # Safety
+///
+/// This function should not be called before the Operator are ready.
+#[no_mangle]
+pub unsafe extern "system" fn Java_org_apache_opendal_Operator_presignRead(
+ mut env: JNIEnv,
+ _: JClass,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> jlong {
+ intern_presign_read(&mut env, op, path, expire).unwrap_or_else(|e| {
+ e.throw(&mut env);
+ 0
+ })
+}
+
+fn intern_presign_read(
+ env: &mut JNIEnv,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> Result<jlong> {
+ let op = unsafe { &mut *op };
+ let id = request_id(env)?;
+
+ let path = env.get_string(&path)?.to_str()?.to_string();
+ let expire = Duration::from_nanos(expire as u64);
+
+ unsafe { get_global_runtime() }.spawn(async move {
+ let result = do_presign_read(op, path, expire).await;
+ let mut env = unsafe { get_current_env() };
+ let result = result.and_then(|req| make_presigned_request(&mut env,
req));
+ complete_future(id, result.map(JValueOwned::Object))
+ });
+
+ Ok(id)
+}
+
+async fn do_presign_read(
+ op: &mut Operator,
+ path: String,
+ expire: Duration,
+) -> Result<PresignedRequest> {
+ Ok(op.presign_read(&path, expire).await?)
+}
+
+/// # Safety
+///
+/// This function should not be called before the Operator are ready.
+#[no_mangle]
+pub unsafe extern "system" fn Java_org_apache_opendal_Operator_presignWrite(
+ mut env: JNIEnv,
+ _: JClass,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> jlong {
+ intern_presign_write(&mut env, op, path, expire).unwrap_or_else(|e| {
+ e.throw(&mut env);
+ 0
+ })
+}
+
+fn intern_presign_write(
+ env: &mut JNIEnv,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> Result<jlong> {
+ let op = unsafe { &mut *op };
+ let id = request_id(env)?;
+
+ let path = env.get_string(&path)?.to_str()?.to_string();
+ let expire = Duration::from_nanos(expire as u64);
+
+ unsafe { get_global_runtime() }.spawn(async move {
+ let result = do_presign_write(op, path, expire).await;
+ let mut env = unsafe { get_current_env() };
+ let result = result.and_then(|req| make_presigned_request(&mut env,
req));
+ complete_future(id, result.map(JValueOwned::Object))
+ });
+
+ Ok(id)
+}
+
+async fn do_presign_write(
+ op: &mut Operator,
+ path: String,
+ expire: Duration,
+) -> Result<PresignedRequest> {
+ Ok(op.presign_write(&path, expire).await?)
+}
+
+/// # Safety
+///
+/// This function should not be called before the Operator are ready.
+#[no_mangle]
+pub unsafe extern "system" fn Java_org_apache_opendal_Operator_presignStat(
+ mut env: JNIEnv,
+ _: JClass,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> jlong {
+ intern_presign_stat(&mut env, op, path, expire).unwrap_or_else(|e| {
+ e.throw(&mut env);
+ 0
+ })
+}
+
+fn intern_presign_stat(
+ env: &mut JNIEnv,
+ op: *mut Operator,
+ path: JString,
+ expire: jlong,
+) -> Result<jlong> {
+ let op = unsafe { &mut *op };
+ let id = request_id(env)?;
+
+ let path = env.get_string(&path)?.to_str()?.to_string();
+ let expire = Duration::from_nanos(expire as u64);
+
+ unsafe { get_global_runtime() }.spawn(async move {
+ let result = do_presign_stat(op, path, expire).await;
+ let mut env = unsafe { get_current_env() };
+ let result = result.and_then(|req| make_presigned_request(&mut env,
req));
+ complete_future(id, result.map(JValueOwned::Object))
+ });
+
+ Ok(id)
+}
+
+async fn do_presign_stat(
+ op: &mut Operator,
+ path: String,
+ expire: Duration,
+) -> Result<PresignedRequest> {
+ Ok(op.presign_stat(&path, expire).await?)
+}
+
fn make_object<'local>(
env: &mut JNIEnv<'local>,
value: JValueOwned<'local>,
diff --git a/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java
b/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java
index 996c79080..5955d6c32 100644
--- a/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/AsyncStepsTest.java
@@ -19,15 +19,18 @@
package org.apache.opendal;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import io.cucumber.java.en.Given;
import io.cucumber.java.en.Then;
import io.cucumber.java.en.When;
+import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import lombok.Cleanup;
+import org.apache.opendal.condition.OpenDALExceptionCondition;
public class AsyncStepsTest {
Operator op;
@@ -70,6 +73,14 @@ public class AsyncStepsTest {
@Then("The presign operation should success or raise exception
Unsupported")
public void
the_presign_operation_should_success_or_raise_exception_unsupported() {
- // TODO: please implement me
+ assertThatThrownBy(
+ () -> op.presignStat("test.txt",
Duration.ofSeconds(10)).join())
+
.is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported));
+ assertThatThrownBy(
+ () -> op.presignRead("test.txt",
Duration.ofSeconds(10)).join())
+
.is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported));
+ assertThatThrownBy(() ->
+ op.presignWrite("test.txt",
Duration.ofSeconds(10)).join())
+
.is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.Unsupported));
}
}
diff --git a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java
b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java
index 9e81e535e..bcde1c181 100644
--- a/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/OperatorTest.java
@@ -20,13 +20,14 @@
package org.apache.opendal;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.CompletionException;
import java.util.stream.Collectors;
import lombok.Cleanup;
+import org.apache.opendal.condition.OpenDALExceptionCondition;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -43,15 +44,8 @@ public class OperatorTest {
op.write("testCreateAndDelete", "Odin").join();
assertThat(op.read("testCreateAndDelete").join()).isEqualTo("Odin");
op.delete("testCreateAndDelete").join();
- op.stat("testCreateAndDelete")
- .handle((r, e) -> {
- assertThat(r).isNull();
-
assertThat(e).isInstanceOf(CompletionException.class).hasCauseInstanceOf(OpenDALException.class);
- OpenDALException.Code code = ((OpenDALException)
e.getCause()).getCode();
- assertThat(code).isEqualTo(OpenDALException.Code.NotFound);
- return null;
- })
- .join();
+ assertThatThrownBy(() -> op.stat("testCreateAndDelete").join())
+
.is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound));
}
@Test
diff --git
a/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java
b/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java
index b93a40115..a344f2c93 100644
--- a/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java
+++ b/bindings/java/src/test/java/org/apache/opendal/RedisServiceTest.java
@@ -21,10 +21,11 @@ package org.apache.opendal;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.CompletionException;
import lombok.Cleanup;
+import org.apache.opendal.condition.OpenDALExceptionCondition;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.DisabledIfEnvironmentVariable;
import org.testcontainers.containers.GenericContainer;
@@ -49,15 +50,8 @@ public class RedisServiceTest {
op.write("testAccessRedisService", "Odin").join();
assertThat(op.read("testAccessRedisService").join()).isEqualTo("Odin");
op.delete("testAccessRedisService").join();
- op.stat("testAccessRedisService")
- .handle((r, e) -> {
- assertThat(r).isNull();
-
assertThat(e).isInstanceOf(CompletionException.class).hasCauseInstanceOf(OpenDALException.class);
- OpenDALException.Code code = ((OpenDALException)
e.getCause()).getCode();
- assertThat(code).isEqualTo(OpenDALException.Code.NotFound);
- return null;
- })
- .join();
+ assertThatThrownBy(() -> op.stat("testAccessRedisService").join())
+
.is(OpenDALExceptionCondition.ofAsync(OpenDALException.Code.NotFound));
}
@Test
diff --git
a/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java
b/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java
new file mode 100644
index 000000000..4c3afddb8
--- /dev/null
+++
b/bindings/java/src/test/java/org/apache/opendal/condition/OpenDALExceptionCondition.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.opendal.condition;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Predicate;
+import org.apache.opendal.OpenDALException;
+import org.assertj.core.api.Condition;
+
+public class OpenDALExceptionCondition extends Condition<Throwable> {
+ private final List<Class<? extends Throwable>> canBeStripped;
+ private final OpenDALException.Code code;
+
+ public static OpenDALExceptionCondition ofAsync(OpenDALException.Code
code) {
+ final List<Class<? extends Throwable>> canBeStripped = new
ArrayList<>();
+ canBeStripped.add(CompletionException.class);
+ canBeStripped.add(ExecutionException.class);
+ return new OpenDALExceptionCondition(code, canBeStripped);
+ }
+
+ public OpenDALExceptionCondition(OpenDALException.Code code, List<Class<?
extends Throwable>> canBeStripped) {
+ as("OpenDALException with code " + code + ", stripping " +
canBeStripped);
+ this.code = code;
+ this.canBeStripped = canBeStripped;
+ }
+
+ private Throwable stripException(Throwable throwable) {
+ while (throwable.getCause() != null) {
+ final Class<?> thisClass = throwable.getClass();
+ final Predicate<Class<? extends Throwable>> predicate = clazz ->
clazz.isAssignableFrom(thisClass);
+ if (this.canBeStripped.stream().noneMatch(predicate)) {
+ break;
+ }
+ throwable = throwable.getCause();
+ }
+ return throwable;
+ }
+
+ @Override
+ public boolean matches(Throwable throwable) {
+ throwable = stripException(throwable);
+
+ if (throwable instanceof OpenDALException) {
+ final OpenDALException exception = (OpenDALException) throwable;
+ return exception.getCode() == code;
+ }
+
+ return false;
+ }
+}
