(opendal) branch main updated: fix(services/s3): Mark xml deserialize error as temporary during list (#5178)

[xuanwo]

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 5dbe6fe058 fix(services/s3): Mark xml deserialize error as temporary 
during list (#5178)
5dbe6fe058 is described below

commit 5dbe6fe0581e8b5074b4883764a6d02a15f323b7
Author: Xuanwo <git...@xuanwo.io>
AuthorDate: Mon Oct 14 15:32:18 2024 +0800

    fix(services/s3): Mark xml deserialize error as temporary during list 
(#5178)
    
    Signed-off-by: Xuanwo <git...@xuanwo.io>
---
 core/src/services/s3/lister.rs | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs
index 621d82c224..9063e61044 100644
--- a/core/src/services/s3/lister.rs
+++ b/core/src/services/s3/lister.rs
@@ -23,6 +23,7 @@ use super::error::parse_error;
 use crate::raw::oio::PageContext;
 use crate::raw::*;
 use crate::EntryMode;
+use crate::Error;
 use crate::Metadata;
 use crate::Result;
 use bytes::Buf;
@@ -84,8 +85,14 @@ impl oio::PageList for S3Lister {
         }
         let bs = resp.into_body();
 
-        let output: ListObjectsOutput =
-            de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+        let output: ListObjectsOutput = de::from_reader(bs.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow S3 list to retry on XML deserialization errors.
+            //
+            // This is because the S3 list API may return incomplete XML data 
under high load.
+            // We are confident that our XML decoding logic is correct. When 
this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
 
         // Try our best to check whether this list is done.
         //
@@ -196,8 +203,14 @@ impl oio::PageList for S3ObjectVersionsLister {
         }
 
         let body = resp.into_body();
-        let output: ListObjectVersionsOutput =
-            de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
+        let output: ListObjectVersionsOutput = de::from_reader(body.reader())
+            .map_err(new_xml_deserialize_error)
+            // Allow S3 list to retry on XML deserialization errors.
+            //
+            // This is because the S3 list API may return incomplete XML data 
under high load.
+            // We are confident that our XML decoding logic is correct. When 
this error occurs,
+            // we allow retries to obtain the correct data.
+            .map_err(Error::set_temporary)?;
 
         ctx.done = if let Some(is_truncated) = output.is_truncated {
             !is_truncated