This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 9adefe9462 fix: Azdls returns 403 while continuation contains `=`
(#4105)
9adefe9462 is described below
commit 9adefe946297c53d9375baae690bd37f55bd9b89
Author: Xuanwo <[email protected]>
AuthorDate: Wed Jan 31 20:56:52 2024 +0800
fix: Azdls returns 403 while continuation contains `=` (#4105)
* fix: Azdls returns 403 while continuation contains `=`
Signed-off-by: Xuanwo <[email protected]>
* Don't create too much files
Signed-off-by: Xuanwo <[email protected]>
* Fix build
Signed-off-by: Xuanwo <[email protected]>
* Fix swift
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
Cargo.lock | 5 ++---
bindings/c/Cargo.lock | 5 ++---
core/Cargo.toml | 2 +-
core/src/services/azdls/core.rs | 5 +++--
core/src/services/swift/core.rs | 4 ++++
core/src/services/swift/lister.rs | 15 ++++++++++++---
core/tests/behavior/async_list.rs | 6 +++---
7 files changed, 27 insertions(+), 15 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ab20f0dacc..362ceeeec8 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5872,9 +5872,9 @@ dependencies = [
[[package]]
name = "reqsign"
-version = "0.14.6"
+version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dce87f66ba6c6acef277a729f989a0eca946cb9ce6a15bcc036bda0f72d4b9fd"
+checksum = "ed08ac3aa0676637644b1b892202f1ae789c28c15ebfa906128d111ae8086062"
dependencies = [
"anyhow",
"async-trait",
@@ -5899,7 +5899,6 @@ dependencies = [
"serde_json",
"sha1",
"sha2",
- "tokio",
]
[[package]]
diff --git a/bindings/c/Cargo.lock b/bindings/c/Cargo.lock
index 69571feb89..ef4363c37e 100644
--- a/bindings/c/Cargo.lock
+++ b/bindings/c/Cargo.lock
@@ -1126,9 +1126,9 @@ dependencies = [
[[package]]
name = "reqsign"
-version = "0.14.6"
+version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dce87f66ba6c6acef277a729f989a0eca946cb9ce6a15bcc036bda0f72d4b9fd"
+checksum = "ed08ac3aa0676637644b1b892202f1ae789c28c15ebfa906128d111ae8086062"
dependencies = [
"anyhow",
"async-trait",
@@ -1153,7 +1153,6 @@ dependencies = [
"serde_json",
"sha1",
"sha2",
- "tokio",
]
[[package]]
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 908e189141..81a1668eee 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -274,7 +274,7 @@ sha1 = { version = "0.10.6", optional = true }
sha2 = { version = "0.10", optional = true }
# For http based services.
-reqsign = { version = "0.14.6", default-features = false, optional = true }
+reqsign = { version = "0.14.7", default-features = false, optional = true }
# for services-atomic-server
atomic_lib = { version = "0.34.5", optional = true }
diff --git a/core/src/services/azdls/core.rs b/core/src/services/azdls/core.rs
index d17296f9cb..dce6e5bb80 100644
--- a/core/src/services/azdls/core.rs
+++ b/core/src/services/azdls/core.rs
@@ -294,10 +294,11 @@ impl AzdlsCore {
.expect("write into string must succeed");
}
if let Some(limit) = limit {
- write!(url, "&maxresults={limit}").expect("write into string must
succeed");
+ write!(url, "&maxResults={limit}").expect("write into string must
succeed");
}
if !continuation.is_empty() {
- write!(url, "&continuation={continuation}").expect("write into
string must succeed");
+ write!(url, "&continuation={}", percent_encode_path(continuation))
+ .expect("write into string must succeed");
}
let mut req = Request::get(&url)
diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs
index ca98a518ee..8753c964fb 100644
--- a/core/src/services/swift/core.rs
+++ b/core/src/services/swift/core.rs
@@ -73,6 +73,7 @@ impl SwiftCore {
path: &str,
delimiter: &str,
limit: Option<usize>,
+ marker: &str,
) -> Result<Response<IncomingAsyncBody>> {
let p = build_abs_path(&self.root, path);
@@ -90,6 +91,9 @@ impl SwiftCore {
if let Some(limit) = limit {
url += &format!("&limit={}", limit);
}
+ if !marker.is_empty() {
+ url += &format!("&marker={}", marker);
+ }
let mut req = Request::get(&url);
diff --git a/core/src/services/swift/lister.rs
b/core/src/services/swift/lister.rs
index 2993269614..464f36a9c2 100644
--- a/core/src/services/swift/lister.rs
+++ b/core/src/services/swift/lister.rs
@@ -48,7 +48,7 @@ impl oio::PageList for SwiftLister {
async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> {
let response = self
.core
- .swift_list(&self.path, self.delimiter, self.limit)
+ .swift_list(&self.path, self.delimiter, self.limit, &ctx.token)
.await?;
let status_code = response.status();
@@ -58,12 +58,21 @@ impl oio::PageList for SwiftLister {
return Err(error);
}
- ctx.done = true;
-
let bytes = response.into_body().bytes().await?;
let decoded_response: Vec<ListOpResponse> =
serde_json::from_slice(&bytes).map_err(new_json_deserialize_error)?;
+ // Update token and done based on resp.
+ if let Some(entry) = decoded_response.last() {
+ let path = match entry {
+ ListOpResponse::Subdir { subdir } => subdir,
+ ListOpResponse::FileInfo { name, .. } => name,
+ };
+ ctx.token = path.clone();
+ } else {
+ ctx.done = true;
+ }
+
for status in decoded_response {
let entry: oio::Entry = match status {
ListOpResponse::Subdir { subdir } => {
diff --git a/core/tests/behavior/async_list.rs
b/core/tests/behavior/async_list.rs
index 2d22183e52..b526e39e06 100644
--- a/core/tests/behavior/async_list.rs
+++ b/core/tests/behavior/async_list.rs
@@ -207,7 +207,7 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()>
{
op.create_dir("test_list_rich_dir/").await?;
- let mut expected: Vec<String> = (0..=100)
+ let mut expected: Vec<String> = (0..=10)
.map(|num| format!("test_list_rich_dir/file-{num}"))
.collect();
@@ -215,7 +215,7 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()>
{
op.write(path, "test_list_rich_dir").await?;
}
- let mut objects = op.with_limit(10).lister("test_list_rich_dir/").await?;
+ let mut objects = op.lister_with("test_list_rich_dir/").limit(5).await?;
let mut actual = vec![];
while let Some(o) = objects.try_next().await? {
let path = o.path().to_string();
@@ -228,8 +228,8 @@ pub async fn test_list_rich_dir(op: Operator) -> Result<()>
{
// List concurrently.
let mut objects = op
- .with_limit(10)
.lister_with("test_list_rich_dir/")
+ .limit(5)
.concurrent(5)
.metakey(Metakey::Complete)
.await?;
