This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 8c78dcf5a2 feat(bindings/c): add writer operation for Bindings C and
Go (#5141)
8c78dcf5a2 is described below
commit 8c78dcf5a2462e3663b9644267aa3e3937cfd3bc
Author: Hanchin Hsieh <m...@yuchanns.xyz>
AuthorDate: Wed Sep 25 17:01:18 2024 +0800
feat(bindings/c): add writer operation for Bindings C and Go (#5141)
* feat(bindings/c): add writer operation
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
* feat(bindings/go): add writer operation
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
* ci(bindings/go): test against service-fs
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
* fix(bindings/go): fix tests
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
* fix(bindings/c): use BlockWriter instead of StdWriter
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
---------
Signed-off-by: Hanchin Hsieh <m...@yuchanns.xyz>
---
.github/scripts/test_go_binding/matrix.yaml | 2 +-
.github/workflows/ci_bindings_go.yml | 1 +
bindings/c/include/opendal.h | 100 ++++++++++++++
bindings/c/src/lib.rs | 5 +
bindings/c/src/operator.rs | 63 +++++++++
bindings/c/src/result.rs | 22 +++
bindings/c/src/writer.rs | 70 ++++++++++
bindings/c/tests/bdd.cpp | 21 ++-
bindings/go/ffi.go | 4 +
bindings/go/tests/behavior_tests/go.mod | 5 +-
bindings/go/tests/behavior_tests/go.sum | 10 +-
bindings/go/tests/behavior_tests/list_test.go | 14 +-
bindings/go/tests/behavior_tests/opendal_test.go | 6 +-
bindings/go/tests/behavior_tests/write_test.go | 30 +++++
bindings/go/types.go | 33 +++++
bindings/go/write.go | 162 ++++++++++++++++++++++-
16 files changed, 526 insertions(+), 22 deletions(-)
diff --git a/.github/scripts/test_go_binding/matrix.yaml
b/.github/scripts/test_go_binding/matrix.yaml
index 27ba73a850..e477aea073 100644
--- a/.github/scripts/test_go_binding/matrix.yaml
+++ b/.github/scripts/test_go_binding/matrix.yaml
@@ -22,5 +22,5 @@ build:
goos: "linux"
goarch: "amd64"
service:
- - "memory"
+ - "fs"
diff --git a/.github/workflows/ci_bindings_go.yml
b/.github/workflows/ci_bindings_go.yml
index bde9e0cb2d..5ffd5dca56 100644
--- a/.github/workflows/ci_bindings_go.yml
+++ b/.github/workflows/ci_bindings_go.yml
@@ -127,5 +127,6 @@ jobs:
- name: Run tests
env:
OPENDAL_TEST: ${{ matrix.service }}
+ OPENDAL_FS_ROOT: "/tmp/opendal/"
working-directory: bindings/go/tests/behavior_tests
run: CGO_ENABLE=0 go test -v -run TestBehavior
diff --git a/bindings/c/include/opendal.h b/bindings/c/include/opendal.h
index 585d0be023..2337b84d0a 100644
--- a/bindings/c/include/opendal.h
+++ b/bindings/c/include/opendal.h
@@ -146,6 +146,12 @@ typedef struct BlockingLister BlockingLister;
*/
typedef struct BlockingOperator BlockingOperator;
+/**
+ * BlockingWriter is designed to write data into given path in an blocking
+ * manner.
+ */
+typedef struct BlockingWriter BlockingWriter;
+
/**
* Entry returned by [`Lister`] or [`BlockingLister`] to represent a path and
it's relative metadata.
*
@@ -430,6 +436,31 @@ typedef struct opendal_result_operator_reader {
struct opendal_error *error;
} opendal_result_operator_reader;
+/**
+ * \brief The result type returned by opendal's writer operation.
+ * \note The opendal_writer actually owns a pointer to
+ * a opendal::BlockingWriter, which is inside the Rust core code.
+ */
+typedef struct opendal_writer {
+ struct BlockingWriter *inner;
+} opendal_writer;
+
+/**
+ * \brief The result type returned by opendal_operator_writer().
+ * The result type for opendal_operator_writer(), the field `writer` contains
the writer
+ * of the path, which is an iterator of the objects under the path. the field
`code` represents
+ */
+typedef struct opendal_result_operator_writer {
+ /**
+ * The pointer for opendal_writer
+ */
+ struct opendal_writer *writer;
+ /**
+ * The error, if ok, it is null
+ */
+ struct opendal_error *error;
+} opendal_result_operator_writer;
+
/**
* \brief The result type returned by opendal_operator_is_exist().
*
@@ -680,6 +711,22 @@ typedef struct opendal_result_reader_read {
struct opendal_error *error;
} opendal_result_reader_read;
+/**
+ * \brief The result type returned by opendal_writer_write().
+ * The result type contains a size field, which is the size of the data
written,
+ * which is zero on error. The error field is the error code and error message.
+ */
+typedef struct opendal_result_writer_write {
+ /**
+ * The write size if succeed.
+ */
+ uintptr_t size;
+ /**
+ * The error, if ok, it is null
+ */
+ struct opendal_error *error;
+} opendal_result_writer_write;
+
#ifdef __cplusplus
extern "C" {
#endif // __cplusplus
@@ -970,6 +1017,47 @@ struct opendal_result_read opendal_operator_read(const
struct opendal_operator *
struct opendal_result_operator_reader opendal_operator_reader(const struct
opendal_operator *op,
const char
*path);
+/**
+ * \brief Blockingly create a writer for the specified path.
+ *
+ * This function prepares a writer that can be used to write data to the
specified path
+ * using the provided operator. If successful, it returns a valid writer;
otherwise, it
+ * returns an error.
+ *
+ * @param op The opendal_operator created previously
+ * @param path The designated path where the writer will be used
+ * @see opendal_operator
+ * @see opendal_result_operator_writer
+ * @see opendal_error
+ * @return Returns opendal_result_operator_writer, containing a writer and an
opendal_error.
+ * If the operation succeeds, the `writer` field holds a valid writer and the
`error` field
+ * is null. Otherwise, the `writer` will be null and the `error` will be set
correspondingly.
+ *
+ * # Example
+ *
+ * Following is an example
+ * ```C
+ * //...prepare your opendal_operator, named op for example
+ *
+ * opendal_result_operator_writer result = opendal_operator_writer(op,
"/testpath");
+ * assert(result.error == NULL);
+ * opendal_writer *writer = result.writer;
+ * // Use the writer to write data...
+ * ```
+ *
+ * # Safety
+ *
+ * It is **safe** under the cases below
+ * * The memory pointed to by `path` must contain a valid nul terminator at
the end of
+ * the string.
+ *
+ * # Panic
+ *
+ * * If the `path` points to NULL, this function panics, i.e. exits with
information
+ */
+struct opendal_result_operator_writer opendal_operator_writer(const struct
opendal_operator *op,
+ const char
*path);
+
/**
* \brief Blockingly delete the object in `path`.
*
@@ -1419,6 +1507,18 @@ struct opendal_result_reader_read
opendal_reader_read(const struct opendal_reade
*/
void opendal_reader_free(struct opendal_reader *ptr);
+/**
+ * \brief Write data to the writer.
+ */
+struct opendal_result_writer_write opendal_writer_write(const struct
opendal_writer *writer,
+ struct opendal_bytes
bytes);
+
+/**
+ * \brief Frees the heap memory used by the opendal_writer.
+ * \note This function make sure all data have been stored.
+ */
+void opendal_writer_free(struct opendal_writer *ptr);
+
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs
index cfd46f92ed..ac125b0d42 100644
--- a/bindings/c/src/lib.rs
+++ b/bindings/c/src/lib.rs
@@ -51,9 +51,11 @@ pub use result::opendal_result_list;
pub use result::opendal_result_lister_next;
pub use result::opendal_result_operator_new;
pub use result::opendal_result_operator_reader;
+pub use result::opendal_result_operator_writer;
pub use result::opendal_result_read;
pub use result::opendal_result_reader_read;
pub use result::opendal_result_stat;
+pub use result::opendal_result_writer_write;
mod types;
pub use types::opendal_bytes;
@@ -64,3 +66,6 @@ pub use entry::opendal_entry;
mod reader;
pub use reader::opendal_reader;
+
+mod writer;
+pub use writer::opendal_writer;
diff --git a/bindings/c/src/operator.rs b/bindings/c/src/operator.rs
index 71a1e52e68..ebfca45fdc 100644
--- a/bindings/c/src/operator.rs
+++ b/bindings/c/src/operator.rs
@@ -391,6 +391,69 @@ pub unsafe extern "C" fn opendal_operator_reader(
}
}
+/// \brief Blockingly create a writer for the specified path.
+///
+/// This function prepares a writer that can be used to write data to the
specified path
+/// using the provided operator. If successful, it returns a valid writer;
otherwise, it
+/// returns an error.
+///
+/// @param op The opendal_operator created previously
+/// @param path The designated path where the writer will be used
+/// @see opendal_operator
+/// @see opendal_result_operator_writer
+/// @see opendal_error
+/// @return Returns opendal_result_operator_writer, containing a writer and an
opendal_error.
+/// If the operation succeeds, the `writer` field holds a valid writer and the
`error` field
+/// is null. Otherwise, the `writer` will be null and the `error` will be set
correspondingly.
+///
+/// # Example
+///
+/// Following is an example
+/// ```C
+/// //...prepare your opendal_operator, named op for example
+///
+/// opendal_result_operator_writer result = opendal_operator_writer(op,
"/testpath");
+/// assert(result.error == NULL);
+/// opendal_writer *writer = result.writer;
+/// // Use the writer to write data...
+/// ```
+///
+/// # Safety
+///
+/// It is **safe** under the cases below
+/// * The memory pointed to by `path` must contain a valid nul terminator at
the end of
+/// the string.
+///
+/// # Panic
+///
+/// * If the `path` points to NULL, this function panics, i.e. exits with
information
+#[no_mangle]
+pub unsafe extern "C" fn opendal_operator_writer(
+ op: *const opendal_operator,
+ path: *const c_char,
+) -> opendal_result_operator_writer {
+ if path.is_null() {
+ panic!("The path given is pointing at NULL");
+ }
+ let op = (*op).as_ref();
+
+ let path = unsafe { std::ffi::CStr::from_ptr(path).to_str().unwrap() };
+ let writer = match op.writer(path) {
+ Ok(writer) => writer,
+ Err(err) => {
+ return opendal_result_operator_writer {
+ writer: std::ptr::null_mut(),
+ error: opendal_error::new(err),
+ }
+ }
+ };
+
+ opendal_result_operator_writer {
+ writer: Box::into_raw(Box::new(opendal_writer::new(writer))),
+ error: std::ptr::null_mut(),
+ }
+}
+
/// \brief Blockingly delete the object in `path`.
///
/// Delete the object in `path` blockingly by `op_ptr`.
diff --git a/bindings/c/src/result.rs b/bindings/c/src/result.rs
index 64b2789228..ba7e1c64a0 100644
--- a/bindings/c/src/result.rs
+++ b/bindings/c/src/result.rs
@@ -131,3 +131,25 @@ pub struct opendal_result_reader_read {
/// The error, if ok, it is null
pub error: *mut opendal_error,
}
+
+/// \brief The result type returned by opendal_operator_writer().
+/// The result type for opendal_operator_writer(), the field `writer` contains
the writer
+/// of the path, which is an iterator of the objects under the path. the field
`code` represents
+#[repr(C)]
+pub struct opendal_result_operator_writer {
+ /// The pointer for opendal_writer
+ pub writer: *mut opendal_writer,
+ /// The error, if ok, it is null
+ pub error: *mut opendal_error,
+}
+
+/// \brief The result type returned by opendal_writer_write().
+/// The result type contains a size field, which is the size of the data
written,
+/// which is zero on error. The error field is the error code and error
message.
+#[repr(C)]
+pub struct opendal_result_writer_write {
+ /// The write size if succeed.
+ pub size: usize,
+ /// The error, if ok, it is null
+ pub error: *mut opendal_error,
+}
diff --git a/bindings/c/src/writer.rs b/bindings/c/src/writer.rs
new file mode 100644
index 0000000000..5113e55c75
--- /dev/null
+++ b/bindings/c/src/writer.rs
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use ::opendal as core;
+
+use super::*;
+
+/// \brief The result type returned by opendal's writer operation.
+/// \note The opendal_writer actually owns a pointer to
+/// a opendal::BlockingWriter, which is inside the Rust core code.
+#[repr(C)]
+pub struct opendal_writer {
+ inner: *mut core::BlockingWriter,
+}
+
+impl opendal_writer {
+ pub(crate) fn new(writer: core::BlockingWriter) -> Self {
+ Self {
+ inner: Box::into_raw(Box::new(writer)),
+ }
+ }
+
+ /// \brief Write data to the writer.
+ #[no_mangle]
+ pub unsafe extern "C" fn opendal_writer_write(
+ writer: *const Self,
+ bytes: opendal_bytes,
+ ) -> opendal_result_writer_write {
+ let inner = unsafe { &mut *(*writer).inner };
+ let size = bytes.len;
+ match inner.write(bytes) {
+ Ok(()) => opendal_result_writer_write {
+ size,
+ error: std::ptr::null_mut(),
+ },
+ Err(e) => opendal_result_writer_write {
+ size: 0,
+ error: opendal_error::new(
+ core::Error::new(core::ErrorKind::Unexpected, "write
failed from writer")
+ .set_source(e),
+ ),
+ },
+ }
+ }
+
+ /// \brief Frees the heap memory used by the opendal_writer.
+ /// \note This function make sure all data have been stored.
+ #[no_mangle]
+ pub unsafe extern "C" fn opendal_writer_free(ptr: *mut opendal_writer) {
+ if !ptr.is_null() {
+ let mut w = unsafe { Box::from_raw((*ptr).inner) };
+ let _ = w.close();
+ let _ = unsafe { Box::from_raw(ptr) };
+ }
+ }
+}
diff --git a/bindings/c/tests/bdd.cpp b/bindings/c/tests/bdd.cpp
index 370c17fd5b..39856e81e5 100644
--- a/bindings/c/tests/bdd.cpp
+++ b/bindings/c/tests/bdd.cpp
@@ -90,6 +90,20 @@ TEST_F(OpendalBddTest, FeatureTest)
EXPECT_EQ(this->content[i], (char)(r.data->data[i]));
}
+ // The blocking file should be deleted
+ error = opendal_operator_delete(this->p, this->path.c_str());
+ EXPECT_EQ(error, nullptr);
+ e = opendal_operator_is_exist(this->p, this->path.c_str());
+ EXPECT_EQ(e.error, nullptr);
+ EXPECT_FALSE(e.is_exist);
+
+ opendal_result_operator_writer writer = opendal_operator_writer(this->p,
this->path.c_str());
+ EXPECT_EQ(writer.error, nullptr);
+ opendal_result_writer_write w = opendal_writer_write(writer.writer, data);
+ EXPECT_EQ(w.error, nullptr);
+ EXPECT_EQ(w.size, this->content.length());
+ opendal_writer_free(writer.writer);
+
// The blocking file "test" must have content "Hello, World!" and read
into buffer
int length = this->content.length();
unsigned char buffer[this->content.length()];
@@ -102,13 +116,6 @@ TEST_F(OpendalBddTest, FeatureTest)
}
opendal_reader_free(reader.reader);
- // The blocking file should be deleted
- error = opendal_operator_delete(this->p, this->path.c_str());
- EXPECT_EQ(error, nullptr);
- e = opendal_operator_is_exist(this->p, this->path.c_str());
- EXPECT_EQ(e.error, nullptr);
- EXPECT_FALSE(e.is_exist);
-
// The deletion operation should be idempotent
error = opendal_operator_delete(this->p, this->path.c_str());
EXPECT_EQ(error, nullptr);
diff --git a/bindings/go/ffi.go b/bindings/go/ffi.go
index 8d5f2d5844..931c345f2a 100644
--- a/bindings/go/ffi.go
+++ b/bindings/go/ffi.go
@@ -140,4 +140,8 @@ var withFFIs = []contextWithFFI{
withOperatorReader,
withReaderRead,
withReaderFree,
+
+ withOperatorWriter,
+ withWriterWrite,
+ withWriterFree,
}
diff --git a/bindings/go/tests/behavior_tests/go.mod
b/bindings/go/tests/behavior_tests/go.mod
index b6c1ff39f0..dae1477ff1 100644
--- a/bindings/go/tests/behavior_tests/go.mod
+++ b/bindings/go/tests/behavior_tests/go.mod
@@ -20,7 +20,8 @@ module opendal_test
go 1.22.5
require (
- github.com/apache/opendal-go-services/memory
v0.0.0-20240719030108-74ff217cfef9
+ github.com/apache/opendal-go-services/fs v0.1.3
+ github.com/apache/opendal-go-services/memory v0.1.3
github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24
github.com/google/uuid v1.6.0
github.com/stretchr/testify v1.9.0
@@ -30,7 +31,7 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/ebitengine/purego v0.7.1 // indirect
github.com/jupiterrider/ffi v0.1.0-beta.9 // indirect
- github.com/klauspost/compress v1.17.9 // indirect
+ github.com/klauspost/compress v1.17.10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/bindings/go/tests/behavior_tests/go.sum
b/bindings/go/tests/behavior_tests/go.sum
index 3594204ba2..d9f6957f13 100644
--- a/bindings/go/tests/behavior_tests/go.sum
+++ b/bindings/go/tests/behavior_tests/go.sum
@@ -1,5 +1,7 @@
-github.com/apache/opendal-go-services/memory
v0.0.0-20240719030108-74ff217cfef9
h1:tV4Xmoa5Zq2RyrygaXreayLz10C6JqTpyvMYAtdshXQ=
-github.com/apache/opendal-go-services/memory
v0.0.0-20240719030108-74ff217cfef9/go.mod
h1:vldOQuikErKA1wfGnqvjAYB9MON/PWTuFIulMCKIQqM=
+github.com/apache/opendal-go-services/fs v0.1.3
h1:k5pA73gKbQ3MHH2envsKhr1cec2spLm2tl/bCyU53j8=
+github.com/apache/opendal-go-services/fs v0.1.3/go.mod
h1:7EnuyeXRuQh+L47rZ7y2OrhYJLlUYvgvFPItM98XJ5s=
+github.com/apache/opendal-go-services/memory v0.1.3
h1:lUe4n4Y9AmwS6a1KV/ZTLyWLtWpRSSuNZHchcW2s+LQ=
+github.com/apache/opendal-go-services/memory v0.1.3/go.mod
h1:vldOQuikErKA1wfGnqvjAYB9MON/PWTuFIulMCKIQqM=
github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24
h1:2fAl+WS/lZMTtP6onlrmDbb3pltf+5xNTc0Aeu9nYWE=
github.com/apache/opendal/bindings/go
v0.0.0-20240719044908-d9d4279b3a24/go.mod
h1:jyMN6M6h0jMDZitnjvB3KPobM+oZiESrFb3XUplLxhI=
github.com/davecgh/go-spew v1.1.1
h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -10,8 +12,8 @@ github.com/google/uuid v1.6.0
h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod
h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jupiterrider/ffi v0.1.0-beta.9
h1:HCeAPTsTFgwvcfavyJwy1L2ANz0c85W+ZE7LfzjZi3A=
github.com/jupiterrider/ffi v0.1.0-beta.9/go.mod
h1:sOp6VJGFaYyr4APi8gwy6g20QNHv5F8Iq1CVbtC900s=
-github.com/klauspost/compress v1.17.9
h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod
h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.10
h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
+github.com/klauspost/compress v1.17.10/go.mod
h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/pmezard/go-difflib v1.0.0
h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod
h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.9.0
h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
diff --git a/bindings/go/tests/behavior_tests/list_test.go
b/bindings/go/tests/behavior_tests/list_test.go
index f4f26acf4c..1f6a4a36aa 100644
--- a/bindings/go/tests/behavior_tests/list_test.go
+++ b/bindings/go/tests/behavior_tests/list_test.go
@@ -201,13 +201,21 @@ func testListNestedDir(assert *require.Assertions, op
*opendal.Operator, fixture
assert.Nil(err)
defer obs.Close()
var paths []string
+ var foundParent bool
+ var foundDir bool
for obs.Next() {
entry := obs.Entry()
paths = append(paths, entry.Path())
- assert.Equal(dir, entry.Path())
+ if entry.Path() == parent {
+ foundParent = true
+ } else if entry.Path() == dir {
+ foundDir = true
+ }
}
assert.Nil(obs.Error())
- assert.Equal(1, len(paths), "parent should only got 1 entry")
+ assert.Equal(2, len(paths), "parent should only got 2 entry")
+ assert.Equal(foundParent, true, "parent should be found in list")
+ assert.Equal(foundDir, true, "dir should be found in list")
obs, err = op.List(dir)
assert.Nil(err)
@@ -215,7 +223,7 @@ func testListNestedDir(assert *require.Assertions, op
*opendal.Operator, fixture
paths = nil
var foundFile bool
var foundDirPath bool
- var foundDir bool
+ foundDir = false
for obs.Next() {
entry := obs.Entry()
paths = append(paths, entry.Path())
diff --git a/bindings/go/tests/behavior_tests/opendal_test.go
b/bindings/go/tests/behavior_tests/opendal_test.go
index 551087c5f3..8991f971a3 100644
--- a/bindings/go/tests/behavior_tests/opendal_test.go
+++ b/bindings/go/tests/behavior_tests/opendal_test.go
@@ -30,6 +30,7 @@ import (
"sync"
"testing"
+ "github.com/apache/opendal-go-services/fs"
"github.com/apache/opendal-go-services/memory"
opendal "github.com/apache/opendal/bindings/go"
"github.com/google/uuid"
@@ -39,6 +40,7 @@ import (
// Add more schemes for behavior tests here.
var schemes = []opendal.Scheme{
memory.Scheme,
+ fs.Scheme,
}
var op *opendal.Operator
@@ -223,8 +225,8 @@ func (f *fixture) Cleanup(assert *require.Assertions) {
f.lock.Lock()
defer f.lock.Unlock()
- for _, path := range f.paths {
- assert.Nil(f.op.Delete(path), "delete must succeed: %s", path)
+ for i := len(f.paths) - 1; i >= 0; i-- {
+ assert.Nil(f.op.Delete(f.paths[i]), "delete must succeed: %s",
f.paths[i])
}
}
diff --git a/bindings/go/tests/behavior_tests/write_test.go
b/bindings/go/tests/behavior_tests/write_test.go
index 69cece7d43..05d626972d 100644
--- a/bindings/go/tests/behavior_tests/write_test.go
+++ b/bindings/go/tests/behavior_tests/write_test.go
@@ -35,6 +35,7 @@ func testsWrite(cap *opendal.Capability) []behaviorTest {
testWriteWithDirPath,
testWriteWithSpecialChars,
testWriteOverwrite,
+ testWriterWrite,
}
}
@@ -99,3 +100,32 @@ func testWriteOverwrite(assert *require.Assertions, op
*opendal.Operator, fixtur
assert.NotEqual(contentOne, bs, "content_one must be overwrote")
assert.Equal(contentTwo, bs, "read content_two")
}
+
+func testWriterWrite(assert *require.Assertions, op *opendal.Operator, fixture
*fixture) {
+ if !op.Info().GetFullCapability().WriteCanMulti() {
+ return
+ }
+
+ path := fixture.NewFilePath()
+ size := uint(5 * 1024 * 1024)
+ contentA := genFixedBytes(size)
+ contentB := genFixedBytes(size)
+
+ w, err := op.Writer(path)
+ assert.Nil(err)
+ _, err = w.Write(contentA)
+ assert.Nil(err)
+ _, err = w.Write(contentB)
+ assert.Nil(err)
+ assert.Nil(w.Close())
+
+ meta, err := op.Stat(path)
+ assert.Nil(err, "stat must succeed")
+ assert.Equal(uint64(size*2), meta.ContentLength())
+
+ bs, err := op.Read(path)
+ assert.Nil(err, "read must succeed")
+ assert.Equal(uint64(size*2), uint64(len(bs)), "read size")
+ assert.Equal(contentA, bs[:size], "read contentA")
+ assert.Equal(contentB, bs[size:], "read contentB")
+}
diff --git a/bindings/go/types.go b/bindings/go/types.go
index 49c0daf5d3..59583a94af 100644
--- a/bindings/go/types.go
+++ b/bindings/go/types.go
@@ -89,6 +89,24 @@ var (
}[0],
}
+ typeResultOperatorWriter = ffi.Type{
+ Type: ffi.Struct,
+ Elements: &[]*ffi.Type{
+ &ffi.TypePointer,
+ &ffi.TypePointer,
+ nil,
+ }[0],
+ }
+
+ typeResultWriterWrite = ffi.Type{
+ Type: ffi.Struct,
+ Elements: &[]*ffi.Type{
+ &ffi.TypePointer,
+ &ffi.TypePointer,
+ nil,
+ }[0],
+ }
+
typeResultReaderRead = ffi.Type{
Type: ffi.Struct,
Elements: &[]*ffi.Type{
@@ -209,6 +227,18 @@ type resultOperatorReader struct {
error *opendalError
}
+type opendalWriter struct{}
+
+type resultOperatorWriter struct {
+ writer *opendalWriter
+ error *opendalError
+}
+
+type resultWriterWrite struct {
+ size uint
+ error *opendalError
+}
+
type resultReaderRead struct {
size uint
error *opendalError
@@ -257,6 +287,9 @@ func toOpendalBytes(data []byte) opendalBytes {
l := len(data)
if l > 0 {
ptr = &data[0]
+ } else {
+ var b byte
+ ptr = &b
}
return opendalBytes{
data: ptr,
diff --git a/bindings/go/write.go b/bindings/go/write.go
index e1d18a1322..d287524416 100644
--- a/bindings/go/write.go
+++ b/bindings/go/write.go
@@ -21,6 +21,7 @@ package opendal
import (
"context"
+ "io"
"unsafe"
"github.com/jupiterrider/ffi"
@@ -98,6 +99,96 @@ func (op *Operator) CreateDir(path string) error {
return createDir(op.inner, path)
}
+// Writer returns a new Writer for the specified path.
+//
+// Writer is a wrapper around the C-binding function `opendal_operator_writer`.
+// It provides a way to obtain a writer for writing data to the storage system.
+//
+// # Parameters
+//
+// - path: The destination path where data will be written.
+//
+// # Returns
+//
+// - *Writer: A pointer to a Writer instance, or an error if the operation
fails.
+//
+// # Example
+//
+// func exampleWriter(op *opendal.Operator) {
+// writer, err := op.Writer("test/")
+// if err != nil {
+// log.Fatal(err)
+// }
+// defer writer.Close()
+// _, err = writer.Write([]byte("Hello opendal writer!"))
+// if err != nil {
+// log.Fatal(err)
+// }
+// }
+//
+// Note: This example assumes proper error handling and import statements.
+func (op *Operator) Writer(path string) (*Writer, error) {
+ getWriter := getFFI[operatorWriter](op.ctx,
+ symOperatorWriter)
+ inner, err := getWriter(op.inner, path)
+ if err != nil {
+ return nil, err
+ }
+ writer := &Writer{
+ inner: inner,
+ ctx: op.ctx,
+ }
+ return writer, nil
+}
+
+type Writer struct {
+ inner *opendalWriter
+ ctx context.Context
+}
+
+// Write writes the given bytes to the specified path.
+//
+// Write is a wrapper around the C-binding function `opendal_operator_write`.
It provides a simplified
+// interface for writing data to the storage. Write can be called multiple
times to write
+// additional data to the same path.
+//
+// The maximum size of the data that can be written in a single call is 256KB.
+//
+// # Parameters
+//
+// - path: The destination path where the bytes will be written.
+// - data: The byte slice containing the data to be written.
+//
+// # Returns
+//
+// - error: An error if the write operation fails, or nil if successful.
+//
+// # Example
+//
+// func exampleWrite(op *opendal.Operator) {
+// err = op.Write("test", []byte("Hello opendal go binding!"))
+// if err != nil {
+// log.Fatal(err)
+// }
+// }
+//
+// Note: This example assumes proper error handling and import statements.
+func (w *Writer) Write(p []byte) (n int, err error) {
+ write := getFFI[writerWrite](w.ctx, symWriterWrite)
+ return write(w.inner, p)
+}
+
+// Close finishes the write and releases the resources associated with the
Writer.
+// It is important to call Close after writing all the data to ensure that the
data is
+// properly flushed and written to the storage. Otherwise, the data may be
lost.
+func (w *Writer) Close() error {
+ free := getFFI[writerFree](w.ctx, symWriterFree)
+ free(w.inner)
+ return nil
+}
+
+var _ io.WriteCloser = (*Writer)(nil)
+
const symOperatorWrite = "opendal_operator_write"
type operatorWrite func(op *opendalOperator, path string, data []byte) error
@@ -113,9 +204,6 @@ var withOperatorWrite = withFFI(ffiOpts{
return err
}
bytes := toOpendalBytes(data)
- if len(data) > 0 {
- bytes.data = &data[0]
- }
var e *opendalError
ffiCall(
unsafe.Pointer(&e),
@@ -150,3 +238,71 @@ var withOperatorCreateDir = withFFI(ffiOpts{
return parseError(ctx, e)
}
})
+
+const symOperatorWriter = "opendal_operator_writer"
+
+type operatorWriter func(op *opendalOperator, path string) (*opendalWriter,
error)
+
+var withOperatorWriter = withFFI(ffiOpts{
+ sym: symOperatorWriter,
+ rType: &typeResultOperatorWriter,
+ aTypes: []*ffi.Type{&ffi.TypePointer, &ffi.TypePointer},
+}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues
...unsafe.Pointer)) operatorWriter {
+ return func(op *opendalOperator, path string) (*opendalWriter, error) {
+ bytePath, err := unix.BytePtrFromString(path)
+ if err != nil {
+ return nil, err
+ }
+ var result resultOperatorWriter
+ ffiCall(
+ unsafe.Pointer(&result),
+ unsafe.Pointer(&op),
+ unsafe.Pointer(&bytePath),
+ )
+ if result.error != nil {
+ return nil, parseError(ctx, result.error)
+ }
+ return result.writer, nil
+ }
+})
+
+const symWriterFree = "opendal_writer_free"
+
+type writerFree func(w *opendalWriter)
+
+var withWriterFree = withFFI(ffiOpts{
+ sym: symWriterFree,
+ rType: &ffi.TypeVoid,
+ aTypes: []*ffi.Type{&ffi.TypePointer},
+}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues
...unsafe.Pointer)) writerFree {
+ return func(r *opendalWriter) {
+ ffiCall(
+ nil,
+ unsafe.Pointer(&r),
+ )
+ }
+})
+
+const symWriterWrite = "opendal_writer_write"
+
+type writerWrite func(r *opendalWriter, buf []byte) (size int, err error)
+
+var withWriterWrite = withFFI(ffiOpts{
+ sym: symWriterWrite,
+ rType: &typeResultWriterWrite,
+ aTypes: []*ffi.Type{&ffi.TypePointer, &typeBytes},
+}, func(ctx context.Context, ffiCall func(rValue unsafe.Pointer, aValues
...unsafe.Pointer)) writerWrite {
+ return func(r *opendalWriter, data []byte) (size int, err error) {
+ bytes := toOpendalBytes(data)
+ var result resultWriterWrite
+ ffiCall(
+ unsafe.Pointer(&result),
+ unsafe.Pointer(&r),
+ unsafe.Pointer(&bytes),
+ )
+ if result.error != nil {
+ return 0, parseError(ctx, result.error)
+ }
+ return int(result.size), nil
+ }
+})
