This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 50cf8bd806 Always explicitly disable `gzip` automatic decompression on
reqwest client used by object_store (#6843)
50cf8bd806 is described below
commit 50cf8bd80652ec9710bdcb1de568d52618233441
Author: Phillip LeBlanc <[email protected]>
AuthorDate: Wed Dec 11 23:31:02 2024 +0900
Always explicitly disable `gzip` automatic decompression on reqwest client
used by object_store (#6843)
* Explicitly disable gzip on reqwest client used by object_store
* Add comment
* Add integration test for checking reqwest gzip feature
* Fix lint
* Add comment explaining why gzip feature is enabled
---
object_store/Cargo.toml | 2 ++
object_store/src/client/mod.rs | 4 ++++
object_store/tests/http.rs | 43 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 49 insertions(+)
diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml
index a047336251..bcc8e0b922 100644
--- a/object_store/Cargo.toml
+++ b/object_store/Cargo.toml
@@ -77,6 +77,8 @@ http-body-util = "0.1"
rand = "0.8"
tempfile = "3.1.0"
regex = "1.11.1"
+# The "gzip" feature for reqwest is enabled for an integration test.
+reqwest = { version = "0.12", features = ["gzip"] }
http = "1.1.0"
[[test]]
diff --git a/object_store/src/client/mod.rs b/object_store/src/client/mod.rs
index 76d1c1f22f..1b7ce5aa7a 100644
--- a/object_store/src/client/mod.rs
+++ b/object_store/src/client/mod.rs
@@ -671,6 +671,10 @@ impl ClientOptions {
builder = builder.danger_accept_invalid_certs(true)
}
+ // Reqwest will remove the `Content-Length` header if it is configured
to
+ // transparently decompress the body via the non-default `gzip`
feature.
+ builder = builder.no_gzip();
+
builder
.https_only(!self.allow_http.get()?)
.build()
diff --git a/object_store/tests/http.rs b/object_store/tests/http.rs
new file mode 100644
index 0000000000..a9b3145bb6
--- /dev/null
+++ b/object_store/tests/http.rs
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Tests the HTTP store implementation
+
+#[cfg(feature = "http")]
+use object_store::{http::HttpBuilder, path::Path, GetOptions, GetRange,
ObjectStore};
+
+/// Tests that even when reqwest has the `gzip` feature enabled, the HTTP store
+/// does not error on a missing `Content-Length` header.
+#[tokio::test]
+#[cfg(feature = "http")]
+async fn test_http_store_gzip() {
+ let http_store = HttpBuilder::new()
+
.with_url("https://raw.githubusercontent.com/apache/arrow-rs/refs/heads/main";)
+ .build()
+ .unwrap();
+
+ let _ = http_store
+ .get_opts(
+ &Path::parse("LICENSE.txt").unwrap(),
+ GetOptions {
+ range: Some(GetRange::Bounded(0..100)),
+ ..Default::default()
+ },
+ )
+ .await
+ .unwrap();
+}
