This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new ef98b8593 feat(bindings/go): add benchmark (#6341)
ef98b8593 is described below
commit ef98b859384bab3e64a2f0d9fccb85d16b24b14b
Author: Hanchin Hsieh <[email protected]>
AuthorDate: Sat Jun 28 23:57:58 2025 +0800
feat(bindings/go): add benchmark (#6341)
---
.../actions/test_behavior_binding_go/action.yaml | 3 +
bindings/go/Makefile | 13 +-
bindings/go/README.md | 230 +++++++++++++++------
bindings/go/tests/behavior_tests/benchmark_test.go | 151 +++++++++++---
bindings/go/tests/behavior_tests/go.mod | 3 +
bindings/go/tests/behavior_tests/go.sum | 14 +-
6 files changed, 316 insertions(+), 98 deletions(-)
diff --git a/.github/actions/test_behavior_binding_go/action.yaml
b/.github/actions/test_behavior_binding_go/action.yaml
index 21c9ef7e5..99d9eb42f 100644
--- a/.github/actions/test_behavior_binding_go/action.yaml
+++ b/.github/actions/test_behavior_binding_go/action.yaml
@@ -47,6 +47,9 @@ runs:
fi
cp bindings/go/Makefile .
make tests
+ if [ "${{ inputs.setup }}" == "0_minio_s3" ]; then
+ make bench
+ fi
env:
OPENDAL_TEST: ${{ inputs.service }}
CGO_ENABLE: 0
diff --git a/bindings/go/Makefile b/bindings/go/Makefile
index a415bc058..094b9356d 100644
--- a/bindings/go/Makefile
+++ b/bindings/go/Makefile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-.PHONY: tests clean help check-env generate-services run-tests
+.PHONY: tests bench clean help check-env generate-services run-tests run-bench
# Detect current OS and architecture
OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
@@ -139,10 +139,19 @@ run-tests: generate-services
@echo "Running behavior tests..."
go test ./opendal/bindings/go/tests/behavior_tests -v -run TestBehavior
+# Run benchmarks
+run-bench: generate-services
+ @echo "Running benchmark tests..."
+ go test ./opendal/bindings/go/tests/behavior_tests -v -bench=. -run=^$$
-count=6 -benchmem
+
# Main target: run all tests
tests: check-env run-tests
@echo "All tests completed successfully!"
+# Main target: run benchmarks only
+bench: check-env run-bench
+ @echo "All benchmarks completed successfully!"
+
# Clean all generated files
clean:
@echo "Cleaning all generated files..."
@@ -157,6 +166,7 @@ clean:
help:
@echo "Available targets:"
@echo " tests - Run all tests (requires OPENDAL_TEST
environment variable)"
+ @echo " bench - Run benchmarks only (requires OPENDAL_TEST
environment variable)"
@echo " clean - Clean all generated files"
@echo " help - Show this help message"
@echo ""
@@ -165,4 +175,5 @@ help:
@echo ""
@echo "Example usage:"
@echo " OPENDAL_TEST=fs make tests"
+ @echo " OPENDAL_TEST=fs make bench"
diff --git a/bindings/go/README.md b/bindings/go/README.md
index fc4bad89d..e36559c32 100644
--- a/bindings/go/README.md
+++ b/bindings/go/README.md
@@ -114,76 +114,174 @@ go test -bench .
<details>
<summary>
- A benchmark between purego+libffi vs CGO
+ A benchmark between OpenDAL Go binding and aws-sdk-go on minio S3 compatible
storage
</summary>
-[**purego+libffi**](https://github.com/apache/opendal/commit/bf15cecd5e3be6ecaa7056b5594589c9f4d85673)
(as `new.txt`)
```
goos: linux
-goarch: arm64
-pkg: github.com/apache/opendal/bindings/go
-BenchmarkWrite4KiB-10 1000000 2844 ns/op
-BenchmarkWrite256KiB-10 163346 10092 ns/op
-BenchmarkWrite4MiB-10 12900 99161 ns/op
-BenchmarkWrite16MiB-10 1785 658210 ns/op
-BenchmarkRead4KiB-10 194529 6387 ns/op
-BenchmarkRead256KiB-10 14228 82704 ns/op
-BenchmarkRead4MiB-10 981 1227872 ns/op
-BenchmarkRead16MiB-10 328 3617185 ns/op
+goarch: amd64
+pkg: opendal_test
+cpu: AMD EPYC 7763 64-Core Processor
+BenchmarkWrite
+BenchmarkWrite/4KiB/OpenDAL
+BenchmarkWrite/4KiB/OpenDAL-4 405 3430652 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/OpenDAL-4 313 3781825 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/OpenDAL-4 346 3354802 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/OpenDAL-4 397 3374467 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/OpenDAL-4 364 6171386 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/OpenDAL-4 388 3789795 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4KiB/AWS_S3
+BenchmarkWrite/4KiB/AWS_S3-4 384 3716060 ns/op
62929 B/op 312 allocs/op
+BenchmarkWrite/4KiB/AWS_S3-4 402 3145797 ns/op
62077 B/op 312 allocs/op
+BenchmarkWrite/4KiB/AWS_S3-4 372 3050911 ns/op
61902 B/op 312 allocs/op
+BenchmarkWrite/4KiB/AWS_S3-4 400 3081028 ns/op
61556 B/op 312 allocs/op
+BenchmarkWrite/4KiB/AWS_S3-4 342 3111741 ns/op
61970 B/op 312 allocs/op
+BenchmarkWrite/4KiB/AWS_S3-4 464 2933844 ns/op
61617 B/op 312 allocs/op
+BenchmarkWrite/256KiB/OpenDAL
+BenchmarkWrite/256KiB/OpenDAL-4 228 6253966 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/OpenDAL-4 190 5859882 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/OpenDAL-4 216 6008253 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/OpenDAL-4 200 5958440 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/OpenDAL-4 193 5658798 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/OpenDAL-4 210 6250594 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/256KiB/AWS_S3
+BenchmarkWrite/256KiB/AWS_S3-4 208 5665223 ns/op
90323 B/op 312 allocs/op
+BenchmarkWrite/256KiB/AWS_S3-4 196 5368602 ns/op
88719 B/op 312 allocs/op
+BenchmarkWrite/256KiB/AWS_S3-4 218 6108315 ns/op
90784 B/op 312 allocs/op
+BenchmarkWrite/256KiB/AWS_S3-4 217 5828966 ns/op
90379 B/op 312 allocs/op
+BenchmarkWrite/256KiB/AWS_S3-4 210 6447429 ns/op
90981 B/op 312 allocs/op
+BenchmarkWrite/256KiB/AWS_S3-4 177 6323915 ns/op
91548 B/op 313 allocs/op
+BenchmarkWrite/4MiB/OpenDAL
+BenchmarkWrite/4MiB/OpenDAL-4 62 19049500 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/OpenDAL-4 54 19271521 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/OpenDAL-4 69 17849900 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/OpenDAL-4 73 18117693 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/OpenDAL-4 74 18249816 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/OpenDAL-4 62 19556730 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/4MiB/AWS_S3
+BenchmarkWrite/4MiB/AWS_S3-4 43 29514084 ns/op
92211 B/op 320 allocs/op
+BenchmarkWrite/4MiB/AWS_S3-4 39 29297735 ns/op
92514 B/op 320 allocs/op
+BenchmarkWrite/4MiB/AWS_S3-4 42 28956593 ns/op
92282 B/op 320 allocs/op
+BenchmarkWrite/4MiB/AWS_S3-4 42 29218821 ns/op
92282 B/op 320 allocs/op
+BenchmarkWrite/4MiB/AWS_S3-4 40 28988214 ns/op
91391 B/op 320 allocs/op
+BenchmarkWrite/4MiB/AWS_S3-4 43 28668319 ns/op
91242 B/op 320 allocs/op
+BenchmarkWrite/16MiB/OpenDAL
+BenchmarkWrite/16MiB/OpenDAL-4 21 53528117 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/OpenDAL-4 21 55328986 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/OpenDAL-4 21 54221620 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/OpenDAL-4 20 54044030 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/OpenDAL-4 21 53692610 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/OpenDAL-4 21 53288370 ns/op
384 B/op 10 allocs/op
+BenchmarkWrite/16MiB/AWS_S3
+BenchmarkWrite/16MiB/AWS_S3-4 12 99294840 ns/op
99850 B/op 322 allocs/op
+BenchmarkWrite/16MiB/AWS_S3-4 12 97405067 ns/op
99850 B/op 322 allocs/op
+BenchmarkWrite/16MiB/AWS_S3-4 12 97906212 ns/op
99848 B/op 322 allocs/op
+BenchmarkWrite/16MiB/AWS_S3-4 12 98766864 ns/op
96378 B/op 322 allocs/op
+BenchmarkWrite/16MiB/AWS_S3-4 12 97967605 ns/op
99850 B/op 322 allocs/op
+BenchmarkWrite/16MiB/AWS_S3-4 12 97842268 ns/op
96376 B/op 322 allocs/op
+BenchmarkRead
+BenchmarkRead/4KiB/OpenDAL
+BenchmarkRead/4KiB/OpenDAL-4 972 1217197 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/OpenDAL-4 985 1226752 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/OpenDAL-4 982 1216827 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/OpenDAL-4 987 1227682 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/OpenDAL-4 987 1214420 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/OpenDAL-4 987 1215074 ns/op
4760 B/op 15 allocs/op
+BenchmarkRead/4KiB/AWS_S3
+BenchmarkRead/4KiB/AWS_S3-4 1594 759864 ns/op
34988 B/op 344 allocs/op
+BenchmarkRead/4KiB/AWS_S3-4 1605 751628 ns/op
34990 B/op 344 allocs/op
+BenchmarkRead/4KiB/AWS_S3-4 1620 744649 ns/op
34991 B/op 344 allocs/op
+BenchmarkRead/4KiB/AWS_S3-4 1623 752885 ns/op
34994 B/op 344 allocs/op
+BenchmarkRead/4KiB/AWS_S3-4 1629 742307 ns/op
34991 B/op 344 allocs/op
+BenchmarkRead/4KiB/AWS_S3-4 1622 750455 ns/op
34992 B/op 344 allocs/op
+BenchmarkRead/256KiB/OpenDAL
+BenchmarkRead/256KiB/OpenDAL-4 741 1612259 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/OpenDAL-4 727 1633753 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/OpenDAL-4 751 1619770 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/OpenDAL-4 756 1619435 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/OpenDAL-4 750 1629211 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/OpenDAL-4 757 1611646 ns/op
262812 B/op 15 allocs/op
+BenchmarkRead/256KiB/AWS_S3
+BenchmarkRead/256KiB/AWS_S3-4 736 1655133 ns/op
1209847 B/op 365 allocs/op
+BenchmarkRead/256KiB/AWS_S3-4 735 1673714 ns/op
1209874 B/op 365 allocs/op
+BenchmarkRead/256KiB/AWS_S3-4 735 1653903 ns/op
1209852 B/op 365 allocs/op
+BenchmarkRead/256KiB/AWS_S3-4 728 1643222 ns/op
1209855 B/op 365 allocs/op
+BenchmarkRead/256KiB/AWS_S3-4 726 1674202 ns/op
1209842 B/op 365 allocs/op
+BenchmarkRead/256KiB/AWS_S3-4 717 1656092 ns/op
1209876 B/op 365 allocs/op
+BenchmarkRead/4MiB/OpenDAL
+BenchmarkRead/4MiB/OpenDAL-4 250 4933426 ns/op
4194991 B/op 15 allocs/op
+BenchmarkRead/4MiB/OpenDAL-4 255 4874283 ns/op
4194991 B/op 15 allocs/op
+BenchmarkRead/4MiB/OpenDAL-4 252 4769956 ns/op
4194992 B/op 16 allocs/op
+BenchmarkRead/4MiB/OpenDAL-4 244 4800674 ns/op
4194992 B/op 16 allocs/op
+BenchmarkRead/4MiB/OpenDAL-4 242 4826974 ns/op
4194992 B/op 15 allocs/op
+BenchmarkRead/4MiB/OpenDAL-4 254 4882666 ns/op
4194996 B/op 16 allocs/op
+BenchmarkRead/4MiB/AWS_S3
+BenchmarkRead/4MiB/AWS_S3-4 122 10379507 ns/op
21133764 B/op 388 allocs/op
+BenchmarkRead/4MiB/AWS_S3-4 100 10838399 ns/op
21133778 B/op 389 allocs/op
+BenchmarkRead/4MiB/AWS_S3-4 92 11831700 ns/op
21133738 B/op 388 allocs/op
+BenchmarkRead/4MiB/AWS_S3-4 100 10002235 ns/op
21133764 B/op 388 allocs/op
+BenchmarkRead/4MiB/AWS_S3-4 100 10416467 ns/op
21133731 B/op 388 allocs/op
+BenchmarkRead/4MiB/AWS_S3-4 100 10452328 ns/op
21133719 B/op 388 allocs/op
+BenchmarkRead/16MiB/OpenDAL
+BenchmarkRead/16MiB/OpenDAL-4 81 14654347 ns/op
16777903 B/op 15 allocs/op
+BenchmarkRead/16MiB/OpenDAL-4 78 14977822 ns/op
16777908 B/op 16 allocs/op
+BenchmarkRead/16MiB/OpenDAL-4 81 14833921 ns/op
16777906 B/op 16 allocs/op
+BenchmarkRead/16MiB/OpenDAL-4 76 14663855 ns/op
16777906 B/op 16 allocs/op
+BenchmarkRead/16MiB/OpenDAL-4 75 14545209 ns/op
16777906 B/op 16 allocs/op
+BenchmarkRead/16MiB/OpenDAL-4 80 14585250 ns/op
16777905 B/op 16 allocs/op
+BenchmarkRead/16MiB/AWS_S3
+BenchmarkRead/16MiB/AWS_S3-4 52 24826707 ns/op
102660545 B/op 398 allocs/op
+BenchmarkRead/16MiB/AWS_S3-4 46 24588321 ns/op
102660529 B/op 397 allocs/op
+BenchmarkRead/16MiB/AWS_S3-4 44 26526367 ns/op
102660527 B/op 397 allocs/op
+BenchmarkRead/16MiB/AWS_S3-4 44 26996660 ns/op
102660522 B/op 397 allocs/op
+BenchmarkRead/16MiB/AWS_S3-4 42 26319939 ns/op
102660552 B/op 398 allocs/op
+BenchmarkRead/16MiB/AWS_S3-4 45 25286454 ns/op
102660542 B/op 398 allocs/op
PASS
-ok
-```
-
-[**CGO**](https://github.com/apache/opendal/commit/9ef494d6df2e9a13c4e5b9b03bcb36ec30c0a7c0)
(as `old.txt`)
-```
-go test -bench=. -tags dynamic .
-goos: linux
-goarch: arm64
-pkg: opendal.apache.org/go
-BenchmarkWrite4KiB-10 241981 4240 ns/op
-BenchmarkWrite256KiB-10 126464 10105 ns/op
-BenchmarkWrite4MiB-10 13443 89578 ns/op
-BenchmarkWrite16MiB-10 1737 646155 ns/op
-BenchmarkRead4KiB-10 53535 20939 ns/op
-BenchmarkRead256KiB-10 9008 132738 ns/op
-BenchmarkRead4MiB-10 576 1846683 ns/op
-BenchmarkRead16MiB-10 230 6305322 ns/op
-PASS
-ok
+ok opendal_test 147.053s
```
**Diff** with [benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat)
```
-benchstat old.txt new.txt
+benchstat aws.txt opendal.txt
goos: linux
-goarch: arm64
-pkg: github.com/apache/opendal/bindings/go
- │ new.txt │
- │ sec/op │
-Write4KiB-10 2.844µ ± ∞ ¹
-Write256KiB-10 10.09µ ± ∞ ¹
-Write4MiB-10 99.16µ ± ∞ ¹
-Write16MiB-10 658.2µ ± ∞ ¹
-Read4KiB-10 6.387µ ± ∞ ¹
-Read256KiB-10 82.70µ ± ∞ ¹
-Read4MiB-10 1.228m ± ∞ ¹
-Read16MiB-10 3.617m ± ∞ ¹
-geomean 90.23µ
-¹ need >= 6 samples for confidence interval at level 0.95
-
-pkg: opendal.apache.org/go
- │ old.txt │
- │ sec/op │
-Write4KiB-10 4.240µ ± ∞ ¹
-Write256KiB-10 10.11µ ± ∞ ¹
-Write4MiB-10 89.58µ ± ∞ ¹
-Write16MiB-10 646.2µ ± ∞ ¹
-Read4KiB-10 20.94µ ± ∞ ¹
-Read256KiB-10 132.7µ ± ∞ ¹
-Read4MiB-10 1.847m ± ∞ ¹
-Read16MiB-10 6.305m ± ∞ ¹
-geomean 129.7µ
-¹ need >= 6 samples for confidence interval at level 0.95
+goarch: amd64
+pkg: opendal_test
+cpu: AMD EPYC 7763 64-Core Processor
+ │ aws.txt │ opendal.txt │
+ │ sec/op │ sec/op vs base │
+Write/4KiB 3.096m ± 20% 3.606m ± 71% +16.47% (p=0.015 n=6)
+Write/256KiB 5.969m ± 10% 5.983m ± 5% ~ (p=1.000 n=6)
+Write/4MiB 29.10m ± 1% 18.65m ± 5% -35.92% (p=0.002 n=6)
+Write/16MiB 97.94m ± 1% 53.87m ± 3% -45.00% (p=0.002 n=6)
+Read/4KiB 751.0µ ± 1% 1217.0µ ± 1% +62.04% (p=0.002 n=6)
+Read/256KiB 1.656m ± 1% 1.620m ± 1% -2.18% (p=0.002 n=6)
+Read/4MiB 10.434m ± 13% 4.851m ± 2% -53.51% (p=0.002 n=6)
+Read/16MiB 25.80m ± 5% 14.66m ± 2% -43.19% (p=0.002 n=6)
+geomean 8.050m 6.461m -19.73%
+
+ │ aws.txt │ opendal.txt │
+ │ B/op │ B/op vs base │
+Write/4KiB 61936.0 ± 2% 384.0 ± 0% -99.38% (p=0.002 n=6)
+Write/256KiB 90581.5 ± 2% 384.0 ± 0% -99.58% (p=0.002 n=6)
+Write/4MiB 92246.5 ± 1% 384.0 ± 0% -99.58% (p=0.002 n=6)
+Write/16MiB 99849.0 ± 3% 384.0 ± 0% -99.62% (p=0.002 n=6)
+Read/4KiB 34.171Ki ± 0% 4.648Ki ± 0% -86.40% (p=0.002 n=6)
+Read/256KiB 1181.5Ki ± 0% 256.7Ki ± 0% -78.28% (p=0.002 n=6)
+Read/4MiB 20.155Mi ± 0% 4.001Mi ± 0% -80.15% (p=0.002 n=6)
+Read/16MiB 97.90Mi ± 0% 16.00Mi ± 0% -83.66% (p=0.002 n=6)
+geomean 500.3Ki 14.12Ki -97.18%
+
+ │ aws.txt │ opendal.txt │
+ │ allocs/op │ allocs/op vs base │
+Write/4KiB 312.00 ± 0% 10.00 ± 0% -96.79% (p=0.002 n=6)
+Write/256KiB 312.00 ± 0% 10.00 ± 0% -96.79% (p=0.002 n=6)
+Write/4MiB 320.00 ± 0% 10.00 ± 0% -96.88% (p=0.002 n=6)
+Write/16MiB 322.00 ± 0% 10.00 ± 0% -96.89% (p=0.002 n=6)
+Read/4KiB 344.00 ± 0% 15.00 ± 0% -95.64% (p=0.002 n=6)
+Read/256KiB 365.00 ± 0% 15.00 ± 0% -95.89% (p=0.002 n=6)
+Read/4MiB 388.00 ± 0% 15.50 ± 3% -96.01% (p=0.002 n=6)
+Read/16MiB 397.50 ± 0% 16.00 ± 6% -95.97% (p=0.002 n=6)
+geomean 343.6 12.40 -96.39%
```
</details>
@@ -195,10 +293,10 @@ geomean 129.7µ
- [x] IsExist
- [x] Read
- [x] Read
- - [x] Reader -- implement as `io.ReadCloser`
-- [ ] Write
+ - [x] Reader -- implement as `io.ReadSeekCloser`
+- [x] Write
- [x] Write
- - [ ] Writer -- Need support from the C binding
+ - [x] Writer -- implement as `io.WriteCloser`
- [x] Delete
- [x] CreateDir
- [ ] Lister
@@ -256,6 +354,12 @@ make tests
cd -
```
+To run the benchmarks, you can use the following command:
+
+```bash
+make bench
+```
+
</details>
## License and Trademarks
diff --git a/bindings/go/tests/behavior_tests/benchmark_test.go
b/bindings/go/tests/behavior_tests/benchmark_test.go
index f38dabc5a..f53e6566c 100644
--- a/bindings/go/tests/behavior_tests/benchmark_test.go
+++ b/bindings/go/tests/behavior_tests/benchmark_test.go
@@ -20,8 +20,17 @@
package opendal_test
import (
+ "bytes"
+ "fmt"
+ "io"
+ "os"
"testing"
+ opendal "github.com/apache/opendal/bindings/go"
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/s3"
"github.com/google/uuid"
)
@@ -45,7 +54,93 @@ func (s Size) Bytes() uint64 {
return uint64(s)
}
-func runBenchmarkWrite(b *testing.B, size Size) {
+func (s Size) String() string {
+ switch {
+ case s >= MiB:
+ return fmt.Sprintf("%dMiB", s.Bytes()/MiB)
+ case s >= KiB:
+ return fmt.Sprintf("%dKiB", s.Bytes()/KiB)
+ default:
+ return fmt.Sprintf("%dB", s.Bytes())
+ }
+}
+
+var sizes = []Size{
+ fromKibibytes(4),
+ fromKibibytes(256),
+ fromMebibytes(4),
+ fromMebibytes(16),
+}
+
+type ReadWriter interface {
+ Write(path string, data []byte) error
+ Read(path string) ([]byte, error)
+
+ Name() string
+}
+
+type S3ReadWriter struct {
+ client *s3.S3
+}
+
+func NewS3ReadWriter() ReadWriter {
+ s3 := s3.New(session.Must(session.NewSession(&aws.Config{
+ Credentials:
credentials.NewStaticCredentials(os.Getenv("OPENDAL_S3_ACCESS_KEY_ID"),
os.Getenv("OPENDAL_S3_SECRET_ACCESS_KEY"), ""),
+ Endpoint: aws.String(os.Getenv("OPENDAL_S3_ENDPOINT")),
+ Region: aws.String(os.Getenv("OPENDAL_S3_REGION")),
+ S3ForcePathStyle: aws.Bool(true),
+ DisableSSL: aws.Bool(true),
+ })))
+ return &S3ReadWriter{
+ client: s3,
+ }
+}
+
+func (rw *S3ReadWriter) Write(path string, data []byte) error {
+ _, err := rw.client.PutObject(&s3.PutObjectInput{
+ Bucket: aws.String(os.Getenv("OPENDAL_S3_BUCKET")),
+ Key: aws.String(path),
+ Body: aws.ReadSeekCloser(bytes.NewReader(data)),
+ })
+ return err
+}
+
+func (rw *S3ReadWriter) Read(path string) ([]byte, error) {
+ resp, err := rw.client.GetObject(&s3.GetObjectInput{
+ Bucket: aws.String(os.Getenv("OPENDAL_S3_BUCKET")),
+ Key: aws.String(path),
+ })
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ data, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ return data, nil
+}
+
+func (rw *S3ReadWriter) Name() string {
+ return "AWS S3"
+}
+
+type OpenDALReadWriter struct {
+ *opendal.Operator
+}
+
+func NewOpenDALReadWriter(op *opendal.Operator) ReadWriter {
+ return &OpenDALReadWriter{
+ Operator: op,
+ }
+}
+
+func (rw *OpenDALReadWriter) Name() string {
+ return "OpenDAL"
+}
+
+func runBenchmarkWrite(b *testing.B, size Size, op ReadWriter) {
path := uuid.NewString()
data := genFixedBytes(uint(size.Bytes()))
@@ -60,23 +155,21 @@ func runBenchmarkWrite(b *testing.B, size Size) {
}
}
-func BenchmarkWrite4KiB(b *testing.B) {
- runBenchmarkWrite(b, fromKibibytes(4))
-}
-
-func BenchmarkWrite256KiB(b *testing.B) {
- runBenchmarkWrite(b, fromKibibytes(256))
-}
-
-func BenchmarkWrite4MiB(b *testing.B) {
- runBenchmarkWrite(b, fromMebibytes(4))
-}
-
-func BenchmarkWrite16MiB(b *testing.B) {
- runBenchmarkWrite(b, fromMebibytes(16))
+func BenchmarkWrite(b *testing.B) {
+ var ops = []ReadWriter{NewOpenDALReadWriter(op)}
+ if os.Getenv("OPENDAL_TEST") == "s3" {
+ ops = append(ops, NewS3ReadWriter())
+ }
+ for _, size := range sizes {
+ for _, op := range ops {
+ b.Run(fmt.Sprintf("%s/%s", size, op.Name()), func(b
*testing.B) {
+ runBenchmarkWrite(b, size, op)
+ })
+ }
+ }
}
-func runBenchmarkRead(b *testing.B, size Size) {
+func runBenchmarkRead(b *testing.B, size Size, op ReadWriter) {
path := uuid.NewString()
data := genFixedBytes(uint(size.Bytes()))
@@ -97,18 +190,16 @@ func runBenchmarkRead(b *testing.B, size Size) {
}
}
-func BenchmarkRead4KiB(b *testing.B) {
- runBenchmarkRead(b, fromKibibytes(4))
-}
-
-func BenchmarkRead256KiB(b *testing.B) {
- runBenchmarkRead(b, fromKibibytes(256))
-}
-
-func BenchmarkRead4MiB(b *testing.B) {
- runBenchmarkRead(b, fromMebibytes(4))
-}
-
-func BenchmarkRead16MiB(b *testing.B) {
- runBenchmarkRead(b, fromMebibytes(16))
+func BenchmarkRead(b *testing.B) {
+ var ops = []ReadWriter{NewOpenDALReadWriter(op)}
+ if os.Getenv("OPENDAL_TEST") == "s3" {
+ ops = append(ops, NewS3ReadWriter())
+ }
+ for _, size := range sizes {
+ for _, op := range ops {
+ b.Run(fmt.Sprintf("%s/%s", size, op.Name()), func(b
*testing.B) {
+ runBenchmarkRead(b, size, op)
+ })
+ }
+ }
}
diff --git a/bindings/go/tests/behavior_tests/go.mod
b/bindings/go/tests/behavior_tests/go.mod
index fcaba00ec..1a32049bf 100644
--- a/bindings/go/tests/behavior_tests/go.mod
+++ b/bindings/go/tests/behavior_tests/go.mod
@@ -22,6 +22,7 @@ go 1.22.5
require (
github.com/apache/opendal-go-services/fs v0.1.3
github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24
+ github.com/aws/aws-sdk-go v1.55.7
github.com/google/uuid v1.6.0
github.com/stretchr/testify v1.9.0
)
@@ -29,9 +30,11 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/ebitengine/purego v0.7.1 // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jupiterrider/ffi v0.1.0-beta.9 // indirect
github.com/klauspost/compress v1.17.10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.22.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
+
diff --git a/bindings/go/tests/behavior_tests/go.sum
b/bindings/go/tests/behavior_tests/go.sum
index d9f6957f1..c1473d33c 100644
--- a/bindings/go/tests/behavior_tests/go.sum
+++ b/bindings/go/tests/behavior_tests/go.sum
@@ -1,21 +1,25 @@
-github.com/apache/opendal-go-services/fs v0.1.3
h1:k5pA73gKbQ3MHH2envsKhr1cec2spLm2tl/bCyU53j8=
-github.com/apache/opendal-go-services/fs v0.1.3/go.mod
h1:7EnuyeXRuQh+L47rZ7y2OrhYJLlUYvgvFPItM98XJ5s=
-github.com/apache/opendal-go-services/memory v0.1.3
h1:lUe4n4Y9AmwS6a1KV/ZTLyWLtWpRSSuNZHchcW2s+LQ=
-github.com/apache/opendal-go-services/memory v0.1.3/go.mod
h1:vldOQuikErKA1wfGnqvjAYB9MON/PWTuFIulMCKIQqM=
github.com/apache/opendal/bindings/go v0.0.0-20240719044908-d9d4279b3a24
h1:2fAl+WS/lZMTtP6onlrmDbb3pltf+5xNTc0Aeu9nYWE=
github.com/apache/opendal/bindings/go
v0.0.0-20240719044908-d9d4279b3a24/go.mod
h1:jyMN6M6h0jMDZitnjvB3KPobM+oZiESrFb3XUplLxhI=
+github.com/aws/aws-sdk-go v1.55.7
h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE=
+github.com/aws/aws-sdk-go v1.55.7/go.mod
h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/davecgh/go-spew v1.1.0/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1
h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod
h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/ebitengine/purego v0.7.1
h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA=
github.com/ebitengine/purego v0.7.1/go.mod
h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod
h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.4.0
h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod
h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1
h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod
h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jupiterrider/ffi v0.1.0-beta.9
h1:HCeAPTsTFgwvcfavyJwy1L2ANz0c85W+ZE7LfzjZi3A=
github.com/jupiterrider/ffi v0.1.0-beta.9/go.mod
h1:sOp6VJGFaYyr4APi8gwy6g20QNHv5F8Iq1CVbtC900s=
github.com/klauspost/compress v1.17.10
h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
github.com/klauspost/compress v1.17.10/go.mod
h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/pmezard/go-difflib v1.0.0
h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod
h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod
h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.9.0
h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod
h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuchanns/opendal-go-services v0.0.1
h1:qeKv0mOhypQNm97g+u94DnijJK5bdEAp5pdjBGf8N7w=
@@ -24,5 +28,7 @@ golang.org/x/sys v0.22.0
h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405
h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod
h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
