This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new dd594180c feat(core): Make gcs available on wasm32 arch (#3816)
dd594180c is described below
commit dd594180c75e4f6cdc43216f36639d19b97148c3
Author: Xuanwo <[email protected]>
AuthorDate: Mon Dec 25 12:39:04 2023 +0800
feat(core): Make gcs available on wasm32 arch (#3816)
* Fix build on gcs
Signed-off-by: Xuanwo <[email protected]>
* Add CI
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
.github/workflows/ci.yml | 7 +++----
core/Cargo.toml | 2 +-
core/src/raw/oio/write/range_write.rs | 17 +++++++++++------
core/src/services/gcs/backend.rs | 10 ++++++++--
core/src/services/gcs/lister.rs | 3 ++-
core/src/services/gcs/writer.rs | 3 ++-
6 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6bd9a3b84..cdf452be7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -293,7 +293,7 @@ jobs:
)
cargo build --features "${FEATURES[*]}"
- # We only support some services(see `available_services` below) for now, but
we will extend wasm support for other services.
+ # We only support some services(see `available_services` below) for now.
build_under_wasm:
runs-on: ubuntu-latest
steps:
@@ -304,14 +304,13 @@ jobs:
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Build
run: |
- available_services=(
+ FEATURES=(
services-azblob
services-gdrive
services-s3
)
- IFS=","
rustup target add wasm32-unknown-unknown
- cargo build --target wasm32-unknown-unknown --no-default-features
--features="${available_services[*]}"
+ cargo build --target wasm32-unknown-unknown --no-default-features
--features="${FEATURES[*]}"
unit:
runs-on: ubuntu-latest
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 3dd0438ea..111573c41 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -329,4 +329,4 @@ tracing-subscriber = { version = "0.3", features = [
"env-filter",
"tracing-log",
] }
-wiremock = "0.5"
\ No newline at end of file
+wiremock = "0.5"
diff --git a/core/src/raw/oio/write/range_write.rs
b/core/src/raw/oio/write/range_write.rs
index 3872d1cd5..8e6c72f61 100644
--- a/core/src/raw/oio/write/range_write.rs
+++ b/core/src/raw/oio/write/range_write.rs
@@ -20,7 +20,6 @@ use std::task::Context;
use std::task::Poll;
use async_trait::async_trait;
-use futures::future::BoxFuture;
use futures::FutureExt;
use crate::raw::oio::WriteBuf;
@@ -48,7 +47,8 @@ use crate::*;
/// - Services impl `RangeWrite`
/// - `RangeWriter` impl `Write`
/// - Expose `RangeWriter` as `Accessor::Writer`
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
pub trait RangeWrite: Send + Sync + Unpin + 'static {
/// write_once is used to write the data to underlying storage at once.
///
@@ -93,12 +93,17 @@ pub struct RangeWriter<W: RangeWrite> {
enum State<W> {
Idle(Option<W>),
- Init(BoxFuture<'static, (W, Result<String>)>),
- Write(BoxFuture<'static, (W, Result<u64>)>),
- Complete(BoxFuture<'static, (W, Result<()>)>),
- Abort(BoxFuture<'static, (W, Result<()>)>),
+ Init(BoxedFuture<(W, Result<String>)>),
+ Write(BoxedFuture<(W, Result<u64>)>),
+ Complete(BoxedFuture<(W, Result<()>)>),
+ Abort(BoxedFuture<(W, Result<()>)>),
}
+/// # Safety
+///
+/// wasm32 is a special target that we only have one event-loop for this state.
+unsafe impl<S: RangeWrite> Send for State<S> {}
+
/// # Safety
///
/// We will only take `&mut Self` reference for State.
diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs
index 60c8c6b2f..6a0160123 100644
--- a/core/src/services/gcs/backend.rs
+++ b/core/src/services/gcs/backend.rs
@@ -34,8 +34,8 @@ use super::core::*;
use super::error::parse_error;
use super::lister::GcsLister;
use super::writer::GcsWriter;
+use super::writer::GcsWriters;
use crate::raw::*;
-use crate::services::gcs::writer::GcsWriters;
use crate::*;
const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com";;
@@ -269,6 +269,11 @@ impl Builder for GcsBuilder {
if let Some(cred) = &self.config.credential_path {
cred_loader = cred_loader.with_path(cred);
}
+ #[cfg(target_arch = "wasm32")]
+ {
+ cred_loader = cred_loader.with_disable_env();
+ cred_loader = cred_loader.with_disable_well_known_location();
+ }
let scope = if let Some(scope) = &self.config.scope {
scope
@@ -313,7 +318,8 @@ pub struct GcsBackend {
core: Arc<GcsCore>,
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl Accessor for GcsBackend {
type Reader = IncomingAsyncBody;
type BlockingReader = ();
diff --git a/core/src/services/gcs/lister.rs b/core/src/services/gcs/lister.rs
index 218518716..c7da1a86e 100644
--- a/core/src/services/gcs/lister.rs
+++ b/core/src/services/gcs/lister.rs
@@ -60,7 +60,8 @@ impl GcsLister {
}
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl oio::PageList for GcsLister {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
let resp = self
diff --git a/core/src/services/gcs/writer.rs b/core/src/services/gcs/writer.rs
index d55d6d7a9..99b6217f2 100644
--- a/core/src/services/gcs/writer.rs
+++ b/core/src/services/gcs/writer.rs
@@ -43,7 +43,8 @@ impl GcsWriter {
}
}
-#[async_trait]
+#[cfg_attr(not(target_arch = "wasm32"), async_trait)]
+#[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
impl oio::RangeWrite for GcsWriter {
async fn write_once(&self, size: u64, body: AsyncBody) -> Result<()> {
let mut req = self.core.gcs_insert_object_request(
