This is an automated email from the ASF dual-hosted git repository. xuanwo pushed a commit to branch allow-retry-on-incomplete-xml in repository https://gitbox.apache.org/repos/asf/opendal.git
commit c7686c35b03d9a41d289ce9ee39fb62d4b042ae3 Author: Xuanwo <[email protected]> AuthorDate: Mon Oct 14 14:28:03 2024 +0800 fix(services/s3): Mark xml deserialize error as temporary during list Signed-off-by: Xuanwo <[email protected]> --- core/src/services/s3/lister.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs index 621d82c224..9063e61044 100644 --- a/core/src/services/s3/lister.rs +++ b/core/src/services/s3/lister.rs @@ -23,6 +23,7 @@ use super::error::parse_error; use crate::raw::oio::PageContext; use crate::raw::*; use crate::EntryMode; +use crate::Error; use crate::Metadata; use crate::Result; use bytes::Buf; @@ -84,8 +85,14 @@ impl oio::PageList for S3Lister { } let bs = resp.into_body(); - let output: ListObjectsOutput = - de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?; + let output: ListObjectsOutput = de::from_reader(bs.reader()) + .map_err(new_xml_deserialize_error) + // Allow S3 list to retry on XML deserialization errors. + // + // This is because the S3 list API may return incomplete XML data under high load. + // We are confident that our XML decoding logic is correct. When this error occurs, + // we allow retries to obtain the correct data. + .map_err(Error::set_temporary)?; // Try our best to check whether this list is done. // @@ -196,8 +203,14 @@ impl oio::PageList for S3ObjectVersionsLister { } let body = resp.into_body(); - let output: ListObjectVersionsOutput = - de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?; + let output: ListObjectVersionsOutput = de::from_reader(body.reader()) + .map_err(new_xml_deserialize_error) + // Allow S3 list to retry on XML deserialization errors. + // + // This is because the S3 list API may return incomplete XML data under high load. + // We are confident that our XML decoding logic is correct. When this error occurs, + // we allow retries to obtain the correct data. + .map_err(Error::set_temporary)?; ctx.done = if let Some(is_truncated) = output.is_truncated { !is_truncated
