(incubator-opendal) branch main updated: fix(core): Handling content encoding correctly (#3907)

[xuanwo]

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 46f247c196 fix(core): Handling content encoding correctly (#3907)
46f247c196 is described below

commit 46f247c1967018fe49333ff6375d865b362610e6
Author: Xuanwo <git...@xuanwo.io>
AuthorDate: Thu Jan 4 15:16:31 2024 +0800

    fix(core): Handling content encoding correctly (#3907)
    
    * fix(core): Handling content encoding correctly
    
    Signed-off-by: Xuanwo <git...@xuanwo.io>
    
    * polish
    
    Signed-off-by: Xuanwo <git...@xuanwo.io>
    
    ---------
    
    Signed-off-by: Xuanwo <git...@xuanwo.io>
---
 core/src/raw/http_util/client.rs | 9 ++++++---
 core/src/raw/http_util/header.rs | 7 ++++++-
 core/src/raw/http_util/mod.rs    | 1 +
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/core/src/raw/http_util/client.rs b/core/src/raw/http_util/client.rs
index 3bd7e826cd..ea1e9ea0b8 100644
--- a/core/src/raw/http_util/client.rs
+++ b/core/src/raw/http_util/client.rs
@@ -26,6 +26,7 @@ use http::Request;
 use http::Response;
 
 use super::body::IncomingAsyncBody;
+use super::parse_content_encoding;
 use super::parse_content_length;
 use super::AsyncBody;
 use crate::raw::*;
@@ -167,11 +168,13 @@ impl HttpClient {
         })?;
 
         // Get content length from header so that we can check it.
-        // If the request method is HEAD, we will ignore this.
-        let content_length = if is_head {
+        //
+        // - If the request method is HEAD, we will ignore content length.
+        // - If response contains content_encoding, we should omit it's 
content length.
+        let content_length = if is_head || 
parse_content_encoding(resp.headers())?.is_some() {
             None
         } else {
-            parse_content_length(resp.headers()).expect("response content 
length must be valid")
+            parse_content_length(resp.headers())?
         };
 
         let mut hr = Response::builder()
diff --git a/core/src/raw/http_util/header.rs b/core/src/raw/http_util/header.rs
index f266f7c0c4..ec28113a70 100644
--- a/core/src/raw/http_util/header.rs
+++ b/core/src/raw/http_util/header.rs
@@ -19,7 +19,6 @@ use base64::engine::general_purpose;
 use base64::Engine;
 use chrono::DateTime;
 use chrono::Utc;
-use http::header::CACHE_CONTROL;
 use http::header::CONTENT_DISPOSITION;
 use http::header::CONTENT_LENGTH;
 use http::header::CONTENT_RANGE;
@@ -27,6 +26,7 @@ use http::header::CONTENT_TYPE;
 use http::header::ETAG;
 use http::header::LAST_MODIFIED;
 use http::header::LOCATION;
+use http::header::{CACHE_CONTROL, CONTENT_ENCODING};
 use http::HeaderValue;
 use http::{HeaderMap, HeaderName};
 use md5::Digest;
@@ -77,6 +77,11 @@ pub fn parse_content_type(headers: &HeaderMap) -> 
Result<Option<&str>> {
     parse_header_to_str(headers, CONTENT_TYPE)
 }
 
+/// Parse content encoding from header map.
+pub fn parse_content_encoding(headers: &HeaderMap) -> Result<Option<&str>> {
+    parse_header_to_str(headers, CONTENT_ENCODING)
+}
+
 /// Parse content range from header map.
 pub fn parse_content_range(headers: &HeaderMap) -> 
Result<Option<BytesContentRange>> {
     parse_header_to_str(headers, CONTENT_RANGE)?
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index 8dd9a58eea..5a5f375e8d 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -35,6 +35,7 @@ pub use header::format_authorization_by_basic;
 pub use header::format_authorization_by_bearer;
 pub use header::format_content_md5;
 pub use header::parse_content_disposition;
+pub use header::parse_content_encoding;
 pub use header::parse_content_length;
 pub use header::parse_content_md5;
 pub use header::parse_content_range;