This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new f812d2cda Support building `object_store` and `parquet` on
wasm32-unknown-unknown target (#2896)
f812d2cda is described below
commit f812d2cda1379933fca22d9979e3df5be19792fa
Author: John Hughes <[email protected]>
AuthorDate: Wed Oct 26 05:34:18 2022 +0200
Support building `object_store` and `parquet` on wasm32-unknown-unknown
target (#2896)
* Support building object_store on wasm32-unknown-unknown target
* Added cargo check step to parquet workflow for wasm32-unknown-unknown
* Added compile-time warning for unsupported cloud features when compiling
with wasm32
* Added cargo check features to the parquet github workflow.
* Added a section to the README.md for parquet
* * Added wasm32-unknown-unknown section to the object_store README.md
---
.github/actions/setup-builder/action.yaml | 1 +
.github/workflows/parquet.yml | 3 +++
object_store/Cargo.toml | 6 ++++--
object_store/README.md | 9 ++++++++-
object_store/src/lib.rs | 20 +++++++++++++++++---
object_store/src/path/mod.rs | 6 ++++++
object_store/src/util.rs | 1 +
parquet/Cargo.toml | 7 ++++---
parquet/README.md | 8 ++++++++
9 files changed, 52 insertions(+), 9 deletions(-)
diff --git a/.github/actions/setup-builder/action.yaml
b/.github/actions/setup-builder/action.yaml
index 0ef6532da..a4d4d3921 100644
--- a/.github/actions/setup-builder/action.yaml
+++ b/.github/actions/setup-builder/action.yaml
@@ -53,4 +53,5 @@ runs:
echo "Installing ${{ inputs.rust-version }}"
rustup toolchain install ${{ inputs.rust-version }}
rustup default ${{ inputs.rust-version }}
+ rustup target add wasm32-unknown-unknown
echo "CARGO_TARGET_DIR=/github/home/target" >> $GITHUB_ENV
diff --git a/.github/workflows/parquet.yml b/.github/workflows/parquet.yml
index 3c5b2eab7..550b59073 100644
--- a/.github/workflows/parquet.yml
+++ b/.github/workflows/parquet.yml
@@ -119,6 +119,9 @@ jobs:
- name: Check compilation --all-targets --no-default-features
--features json
run: |
cargo check -p parquet --all-targets --no-default-features
--features json
+ - name: Check compilation wasm32-unknown-unknown
+ run: |
+ cargo check -p parquet --no-default-features --features
cli,snap,flate2,brotli --target wasm32-unknown-unknown
clippy:
name: Clippy
diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml
index fc2af7e51..f5eb1115d 100644
--- a/object_store/Cargo.toml
+++ b/object_store/Cargo.toml
@@ -37,7 +37,7 @@ itertools = "0.10.1"
parking_lot = { version = "0.12" }
percent-encoding = "2.1"
snafu = "0.7"
-tokio = { version = "1.18", features = ["sync", "macros", "parking_lot",
"rt-multi-thread", "time", "io-util"] }
+tokio = { version = "1.18", features = ["sync", "macros", "rt", "time",
"io-util"] }
tracing = { version = "0.1" }
url = "2.2"
walkdir = "2"
@@ -51,13 +51,15 @@ rand = { version = "0.8", default-features = false,
features = ["std", "std_rng"
reqwest = { version = "0.11", default-features = false, features =
["rustls-tls"], optional = true }
ring = { version = "0.16", default-features = false, features = ["std"],
optional = true }
rustls-pemfile = { version = "1.0", default-features = false, optional = true }
+# Fix for wasm32-unknown-unknown (see
https://docs.rs/getrandom/latest/getrandom/#webassembly-support)
+getrandom = { version = "0.2", features = ["js"], optional = true }
# AWS Profile support
aws-types = { version = "0.49", optional = true }
aws-config = { version = "0.49", optional = true }
[features]
-cloud = ["serde", "serde_json", "quick-xml", "reqwest", "reqwest/json",
"reqwest/stream", "chrono/serde", "base64", "rand", "ring"]
+cloud = ["serde", "serde_json", "quick-xml", "reqwest", "reqwest/json",
"reqwest/stream", "chrono/serde", "base64", "rand", "ring", "getrandom"]
azure = ["cloud"]
gcp = ["cloud", "rustls-pemfile"]
aws = ["cloud"]
diff --git a/object_store/README.md b/object_store/README.md
index fd10414a9..5b47a65c1 100644
--- a/object_store/README.md
+++ b/object_store/README.md
@@ -33,7 +33,14 @@ change. Supported object stores include:
* Memory
* Custom implementations
-
Originally developed for [InfluxDB
IOx](https://github.com/influxdata/influxdb_iox/) and later split out and
donated to [Apache Arrow](https://arrow.apache.org/).
See [docs.rs](https://docs.rs/object_store) for usage instructions
+
+## Support for `wasm32-unknown-unknown` target
+
+It's possible to build `object_store` for the `wasm32-unknown-unknown` target,
however the cloud storage features `aws`, `azure`, and `gcp` are not supported.
+
+```
+cargo build -p object_store --target wasm32-unknown-unknown
+```
\ No newline at end of file
diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs
index 5eaaabaf2..6278d827b 100644
--- a/object_store/src/lib.rs
+++ b/object_store/src/lib.rs
@@ -153,6 +153,12 @@
//! ```
//!
+#[cfg(all(
+ target_arch = "wasm32",
+ any(feature = "gcp", feature = "aws", feature = "azure",)
+))]
+compile_error!("Features 'gcp', 'aws', 'azure' are not supported on wasm.");
+
#[cfg(feature = "aws")]
pub mod aws;
#[cfg(feature = "azure")]
@@ -160,6 +166,7 @@ pub mod azure;
#[cfg(feature = "gcp")]
pub mod gcp;
pub mod limit;
+#[cfg(not(target_arch = "wasm32"))]
pub mod local;
pub mod memory;
pub mod path;
@@ -176,15 +183,16 @@ mod multipart;
mod util;
use crate::path::Path;
-use crate::util::{
- coalesce_ranges, collect_bytes, maybe_spawn_blocking,
OBJECT_STORE_COALESCE_DEFAULT,
-};
+#[cfg(not(target_arch = "wasm32"))]
+use crate::util::maybe_spawn_blocking;
+use crate::util::{coalesce_ranges, collect_bytes,
OBJECT_STORE_COALESCE_DEFAULT};
use async_trait::async_trait;
use bytes::Bytes;
use chrono::{DateTime, Utc};
use futures::{stream::BoxStream, StreamExt};
use snafu::Snafu;
use std::fmt::{Debug, Formatter};
+#[cfg(not(target_arch = "wasm32"))]
use std::io::{Read, Seek, SeekFrom};
use std::ops::Range;
use tokio::io::AsyncWrite;
@@ -351,6 +359,7 @@ impl GetResult {
/// Collects the data into a [`Bytes`]
pub async fn bytes(self) -> Result<Bytes> {
match self {
+ #[cfg(not(target_arch = "wasm32"))]
Self::File(mut file, path) => {
maybe_spawn_blocking(move || {
let len = file.seek(SeekFrom::End(0)).map_err(|source| {
@@ -377,6 +386,8 @@ impl GetResult {
.await
}
Self::Stream(s) => collect_bytes(s, None).await,
+ #[cfg(target_arch = "wasm32")]
+ _ => unimplemented!("File IO not implemented on wasm32."),
}
}
@@ -396,6 +407,7 @@ impl GetResult {
/// no additional complexity or overheads
pub fn into_stream(self) -> BoxStream<'static, Result<Bytes>> {
match self {
+ #[cfg(not(target_arch = "wasm32"))]
Self::File(file, path) => {
const CHUNK_SIZE: usize = 8 * 1024;
@@ -424,6 +436,8 @@ impl GetResult {
.boxed()
}
Self::Stream(s) => s,
+ #[cfg(target_arch = "wasm32")]
+ _ => unimplemented!("File IO not implemented on wasm32."),
}
}
}
diff --git a/object_store/src/path/mod.rs b/object_store/src/path/mod.rs
index 80e0f792a..59ad471c6 100644
--- a/object_store/src/path/mod.rs
+++ b/object_store/src/path/mod.rs
@@ -18,9 +18,11 @@
//! Path abstraction for Object Storage
use itertools::Itertools;
+#[cfg(not(target_arch = "wasm32"))]
use percent_encoding::percent_decode;
use snafu::{ensure, ResultExt, Snafu};
use std::fmt::Formatter;
+#[cfg(not(target_arch = "wasm32"))]
use url::Url;
/// The delimiter to separate object namespaces, creating a directory
structure.
@@ -160,6 +162,7 @@ impl Path {
})
}
+ #[cfg(not(target_arch = "wasm32"))]
/// Convert a filesystem path to a [`Path`] relative to the filesystem root
///
/// This will return an error if the path contains illegal character
sequences
@@ -176,6 +179,7 @@ impl Path {
Self::from_absolute_path(absolute)
}
+ #[cfg(not(target_arch = "wasm32"))]
/// Convert an absolute filesystem path to a [`Path`] relative to the
filesystem root
///
/// This will return an error if the path contains illegal character
sequences
@@ -184,6 +188,7 @@ impl Path {
Self::from_absolute_path_with_base(path, None)
}
+ #[cfg(not(target_arch = "wasm32"))]
/// Convert a filesystem path to a [`Path`] relative to the provided base
///
/// This will return an error if the path contains illegal character
sequences
@@ -308,6 +313,7 @@ where
}
}
+#[cfg(not(target_arch = "wasm32"))]
/// Given an absolute filesystem path convert it to a URL representation
without canonicalization
pub(crate) fn absolute_path_to_url(
path: impl AsRef<std::path::Path>,
diff --git a/object_store/src/util.rs b/object_store/src/util.rs
index 2814ca244..41c72d012 100644
--- a/object_store/src/util.rs
+++ b/object_store/src/util.rs
@@ -69,6 +69,7 @@ where
}
}
+#[cfg(not(target_arch = "wasm32"))]
/// Takes a function and spawns it to a tokio blocking pool if available
pub async fn maybe_spawn_blocking<F, T>(f: F) -> Result<T>
where
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
index 9c7da94f9..d2c215d46 100644
--- a/parquet/Cargo.toml
+++ b/parquet/Cargo.toml
@@ -30,7 +30,7 @@ edition = "2021"
rust-version = "1.62"
[dependencies]
-ahash = "0.8"
+ahash = { version = "0.8", default-features = false, features =
["compile-time-rng"] }
bytes = { version = "1.1", default-features = false, features = ["std"] }
thrift = { version = "0.16", default-features = false }
snap = { version = "1.0", default-features = false, optional = true }
@@ -46,9 +46,8 @@ base64 = { version = "0.13", default-features = false,
features = ["std"], optio
clap = { version = "4", default-features = false, features = ["std", "derive",
"env", "help", "error-context", "usage"], optional = true }
serde_json = { version = "1.0", default-features = false, features = ["std"],
optional = true }
seq-macro = { version = "0.3", default-features = false }
-rand = { version = "0.8", default-features = false, features = ["std",
"std_rng"] }
futures = { version = "0.3", default-features = false, features = ["std"],
optional = true }
-tokio = { version = "1.0", optional = true, default-features = false, features
= ["macros", "fs", "rt", "io-util"] }
+tokio = { version = "1.0", optional = true, default-features = false, features
= ["macros", "rt", "io-util"] }
hashbrown = { version = "0.12", default-features = false }
[dev-dependencies]
@@ -62,6 +61,8 @@ lz4 = { version = "1.23", default-features = false }
zstd = { version = "0.11", default-features = false }
serde_json = { version = "1.0", features = ["std"], default-features = false }
arrow = { path = "../arrow", version = "25.0.0", default-features = false,
features = ["ipc", "test_utils", "prettyprint", "json"] }
+tokio = { version = "1.0", default-features = false, features = ["macros",
"rt", "io-util", "fs"] }
+rand = { version = "0.8", default-features = false, features = ["std",
"std_rng"] }
[package.metadata.docs.rs]
all-features = true
diff --git a/parquet/README.md b/parquet/README.md
index 96a34d7c2..cd642317a 100644
--- a/parquet/README.md
+++ b/parquet/README.md
@@ -69,6 +69,14 @@ The `parquet` crate provides the following features which
may be enabled in your
- [ ] Predicate pushdown
- [x] Parquet format 4.0.0 support
+## Support for `wasm32-unknown-unknown` target
+
+It's possible to build `parquet` for the `wasm32-unknown-unknown` target,
however not all the compression features are currently unsupported due to
issues with the upstream crates. In particular, the `zstd` and `lz4` features
may have compilation issues. See issue
[#180](https://github.com/apache/arrow-rs/issues/180).
+
+```
+cargo build -p parquet --target wasm32-unknown-unknown --no-default-features
--features cli,snap,flate2,brotli
+```
+
## License
Licensed under the Apache License, Version 2.0:
http://www.apache.org/licenses/LICENSE-2.0.
