This is an automated email from the ASF dual-hosted git repository. suyanhanx pushed a commit to branch nodejs-layer in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
commit 3a6a7562d177d9ee44e4fc8cc7d2bf7b3539b0cd Author: suyanhanx <[email protected]> AuthorDate: Sat Nov 4 14:59:53 2023 +0800 feat(bindings/nodejs): add retry layer Signed-off-by: suyanhanx <[email protected]> --- .github/workflows/bindings_nodejs.yml | 10 ++- bindings/nodejs/Cargo.toml | 1 + bindings/nodejs/generated.js | 3 +- bindings/nodejs/index.d.ts | 15 +++++ bindings/nodejs/index.js | 5 +- bindings/nodejs/src/lib.rs | 120 ++++++++++++++++++++++++++++++++- bindings/nodejs/tests/suites/index.mjs | 11 ++- 7 files changed, 154 insertions(+), 11 deletions(-) diff --git a/.github/workflows/bindings_nodejs.yml b/.github/workflows/bindings_nodejs.yml index 942758dc7..e355c9adf 100644 --- a/.github/workflows/bindings_nodejs.yml +++ b/.github/workflows/bindings_nodejs.yml @@ -36,7 +36,7 @@ on: workflow_dispatch: jobs: - test: + dist-check: runs-on: ubuntu-latest # Notes: this defaults only apply on run tasks. @@ -48,8 +48,6 @@ jobs: - uses: actions/checkout@v4 - name: Setup Rust toolchain uses: ./.github/actions/setup - with: - need-nextest: true - name: Setup node uses: actions/setup-node@v4 with: @@ -65,12 +63,12 @@ jobs: - name: Check format run: yarn run prettier --check . + - name: Build + run: yarn build + - name: Check diff run: git diff --exit-code - - name: Unit test - run: cargo nextest run --no-fail-fast - linux: runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index 9a29697a8..18d65f0e7 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -141,6 +141,7 @@ futures = "0.3.28" napi = { version = "2.11.3", default-features = false, features = [ "napi6", "async", + "compat-mode" ] } napi-derive = "2.12.2" opendal.workspace = true diff --git a/bindings/nodejs/generated.js b/bindings/nodejs/generated.js index 915128b86..8607c1838 100644 --- a/bindings/nodejs/generated.js +++ b/bindings/nodejs/generated.js @@ -271,10 +271,11 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { Operator, Entry, Metadata, Lister, BlockingLister } = nativeBinding +const { Operator, Entry, Metadata, Lister, BlockingLister, RetryLayer } = nativeBinding module.exports.Operator = Operator module.exports.Entry = Entry module.exports.Metadata = Metadata module.exports.Lister = Lister module.exports.BlockingLister = BlockingLister +module.exports.RetryLayer = RetryLayer diff --git a/bindings/nodejs/index.d.ts b/bindings/nodejs/index.d.ts index 112ece450..ce4c5b960 100644 --- a/bindings/nodejs/index.d.ts +++ b/bindings/nodejs/index.d.ts @@ -394,6 +394,8 @@ export class Operator { * ``` */ presignStat(path: string, expires: number): Promise<PresignedRequest> + /** Add a layer to this operator. */ + layer(layer: object): this } export class Entry { /** Return the path of this entry. */ @@ -435,3 +437,16 @@ export class Lister { export class BlockingLister { next(): Entry | null } +/** + * A layer that will retry the request if it fails. + * It will retry with exponential backoff. + * + * ## Parameters + * + * - `jitter`<bool>: Whether to add jitter to the backoff. + * - `max_times`<number>: The maximum number of times to retry. + * - `factor`<number>: The exponential factor to use. + * - `max_delay`<number>: The maximum delay between retries. The unit is microsecond. + * - `min_delay`<number>: The minimum delay between retries. The unit is microsecond. + */ +export class RetryLayer { } diff --git a/bindings/nodejs/index.js b/bindings/nodejs/index.js index 47805a144..197d20950 100644 --- a/bindings/nodejs/index.js +++ b/bindings/nodejs/index.js @@ -20,6 +20,9 @@ /// <reference types="node" /> require('dotenv').config() -const { Operator } = require('./generated.js') +const { Operator, RetryLayer } = require('./generated.js') module.exports.Operator = Operator +module.exports.layers = { + RetryLayer, +} diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs index 7882be4cb..896c91279 100644 --- a/bindings/nodejs/src/lib.rs +++ b/bindings/nodejs/src/lib.rs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +#![allow(clippy::not_unsafe_ptr_arg_deref)] +/// This line exists here because of the `js_function` macro. + #[macro_use] extern crate napi_derive; @@ -24,7 +27,11 @@ use std::time::Duration; use futures::TryStreamExt; use napi::bindgen_prelude::*; -use napi::tokio; + +use napi::CallContext; +use napi::JsBoolean; +use napi::JsNumber; +use napi::JsObject; #[napi] pub struct Operator(opendal::Operator); @@ -685,6 +692,117 @@ impl PresignedRequest { } } +pub trait NodeLayer: Send + Sync { + fn layer(&self, op: opendal::Operator) -> opendal::Operator; +} + +struct NodeLayerWrapper { + inner: Box<dyn NodeLayer>, +} + +#[napi] +impl Operator { + /// Add a layer to this operator. + #[napi] + pub fn layer(&self, env: Env, layer: JsObject) -> Result<Self> { + let ctx: &mut NodeLayerWrapper = env + .unwrap(&layer) + .map_err(|e| Error::from_reason(format!("failed to unwrap layer: {}", e)))?; + Ok(Self(ctx.inner.layer(self.0.clone()))) + } +} + +/// A layer that will retry the request if it fails. +/// It will retry with exponential backoff. +/// +/// ## Parameters +/// +/// - `jitter`<bool>: Whether to add jitter to the backoff. +/// - `max_times`<number>: The maximum number of times to retry. +/// - `factor`<number>: The exponential factor to use. +/// - `max_delay`<number>: The maximum delay between retries. The unit is microsecond. +/// - `min_delay`<number>: The minimum delay between retries. The unit is microsecond. +#[napi] +pub struct RetryLayer(opendal::layers::RetryLayer); + +impl NodeLayer for RetryLayer { + fn layer(&self, op: opendal::Operator) -> opendal::Operator { + op.layer(self.0.clone()) + } +} + +/// RetryLayer constructor. +#[js_function(1)] +pub fn create_retry_layer(ctx: CallContext) -> Result<JsObject> { + let mut retry = opendal::layers::RetryLayer::default(); + + let config: Option<JsObject> = ctx.get::<JsObject>(0).ok(); + + if let Some(config) = config { + let jitter = config.get_named_property::<JsBoolean>("jitter").ok(); + if let Some(jitter) = jitter { + let jitter = jitter.get_value().ok(); + if let Some(jitter) = jitter { + if jitter { + retry = retry.with_jitter(); + } + } + } + + let max_times = config.get_named_property::<JsNumber>("maxTimes").ok(); + if let Some(max_times) = max_times { + let max_times = max_times.get_uint32().ok(); + if let Some(max_times) = max_times { + retry = retry.with_max_times(max_times as usize); + } + } + + let factor = config.get_named_property::<JsNumber>("factor").ok(); + if let Some(factor) = factor { + let factor = factor.get_double().ok(); + if let Some(factor) = factor { + retry = retry.with_factor(factor as f32); + } + } + + let max_delay = config.get_named_property::<JsNumber>("maxDelay").ok(); + if let Some(max_delay) = max_delay { + let max_delay = max_delay.get_uint32().ok(); + if let Some(max_delay) = max_delay { + retry = retry.with_max_delay(Duration::from_millis(max_delay as u64)); + } + } + + let min_delay = config.get_named_property::<JsNumber>("minDelay").ok(); + if let Some(min_delay) = min_delay { + let min_delay = min_delay.get_uint32().ok(); + if let Some(min_delay) = min_delay { + retry = retry.with_min_delay(Duration::from_millis(min_delay as u64)); + } + } + } + + let mut layer = ctx.this_unchecked(); + + ctx.env.wrap( + &mut layer, + NodeLayerWrapper { + inner: Box::new(RetryLayer(retry)), + }, + )?; + + Ok(layer) +} + +/// Export all layers types and constructors to nodejs. +#[module_exports] +pub fn layer_init(mut exports: JsObject, env: Env) -> Result<()> { + let retry_layer = env.define_class("RetryLayer", create_retry_layer, &[])?; + exports.set_named_property("RetryLayer", retry_layer)?; + + Ok(()) +} + fn format_napi_error(err: opendal::Error) -> Error { Error::from_reason(format!("{}", err)) } diff --git a/bindings/nodejs/tests/suites/index.mjs b/bindings/nodejs/tests/suites/index.mjs index cd8998009..a10bc1baf 100644 --- a/bindings/nodejs/tests/suites/index.mjs +++ b/bindings/nodejs/tests/suites/index.mjs @@ -18,7 +18,7 @@ */ import { describe } from 'vitest' -import { Operator } from '../../index.js' +import { Operator, layers } from '../../index.js' import { checkRandomRootEnabled, generateRandomRoot, loadConfigFromEnv } from '../utils.mjs' import { run as AsyncIOTestRun } from './async.suite.mjs' @@ -36,7 +36,14 @@ export function runner(testName, scheme) { config.root = generateRandomRoot(config.root) } - const operator = scheme ? new Operator(scheme, config) : null + let operator = scheme ? new Operator(scheme, config) : null + + let retryLayer = new layers.RetryLayer({ + jitter: true, + maxTimes: 4, + }) + + operator = operator.layer(retryLayer) describe.skipIf(!operator)(testName, () => { AsyncIOTestRun(operator)
