This is an automated email from the ASF dual-hosted git repository.
erickguan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 2e037bbcb feat(services/hf): honor HF_HUB_DISABLE_XET to force http
download mode (#7819)
2e037bbcb is described below
commit 2e037bbcb5d2cec15e284a52705898bf761f9190
Author: Al Johri <[email protected]>
AuthorDate: Tue Jun 30 01:19:12 2026 -0400
feat(services/hf): honor HF_HUB_DISABLE_XET to force http download mode
(#7819)
huggingface_hub documents HF_HUB_DISABLE_XET to disable the hf-xet
integration and fall back to the classic resolve-redirect HTTP download. The
services-huggingface backend already defaults download_mode to Xet and honors
HF_ENDPOINT/HF_HOME, but had no equivalent to HF_HUB_DISABLE_XET.
Resolve download_mode via a new hf_download_mode(): an explicit
.download_mode(...) config still wins; otherwise a set, non-empty
HF_HUB_DISABLE_XET forces Http -- the same convention as the existing
HF_HUB_DISABLE_IMPLICIT_TOKEN check; the default stays Xet. This lets downloads
be forced through HF_ENDPOINT (e.g. a pull-through cache proxy) with no code
change.
Signed-off-by: Al Johri <[email protected]>
---
core/services/hf/src/backend.rs | 58 ++++++++++++++++++++++++++++++++++++++++-
core/services/hf/src/config.rs | 6 +++++
core/services/hf/src/docs.md | 1 +
3 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/core/services/hf/src/backend.rs b/core/services/hf/src/backend.rs
index db9fc0606..fc9411178 100644
--- a/core/services/hf/src/backend.rs
+++ b/core/services/hf/src/backend.rs
@@ -116,6 +116,14 @@ impl HfBuilder {
///
/// - `xet`: uses the XET protocol for downloads (default).
/// - `http`: plain HTTP download, following the redirect from the server.
+ ///
+ /// When this is not set explicitly, the download mode is resolved from the
+ /// `HF_HUB_DISABLE_XET` environment variable (the same variable used by
+ /// `huggingface_hub`): if it is set to a non-empty value, the mode is
forced
+ /// to `http`; otherwise it defaults to `xet`. An explicit value set here
+ /// always takes precedence over the environment variable.
+ ///
+ /// See
<https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
pub fn download_mode(mut self, mode: &str) -> Self {
if !mode.is_empty() {
if let Ok(m) = HfDownloadMode::parse(mode) {
@@ -144,6 +152,20 @@ impl HfBuilder {
.unwrap_or_else(|| "https://huggingface.co".to_string())
}
+ /// Resolve the download mode: an explicit config value wins; otherwise a
set,
+ /// non-empty HF_HUB_DISABLE_XET (a huggingface_hub env var) forces http;
default Xet.
+ fn hf_download_mode(&self) -> HfDownloadMode {
+ if let Some(mode) = self.config.download_mode {
+ return mode;
+ }
+ if let Ok(val) = std::env::var("HF_HUB_DISABLE_XET") {
+ if !val.is_empty() {
+ return HfDownloadMode::Http;
+ }
+ }
+ HfDownloadMode::default()
+ }
+
fn hf_home() -> Option<PathBuf> {
if let Ok(h) = std::env::var("HF_HOME") {
return Some(PathBuf::from(h));
@@ -191,6 +213,7 @@ impl Builder for HfBuilder {
let token = self.hf_token();
let endpoint = self.hf_endpoint();
+ let download_mode = self.hf_download_mode();
let repo_type = self.config.repo_type.ok_or_else(|| {
Error::new(ErrorKind::ConfigInvalid, "repo_type is required")
@@ -216,7 +239,6 @@ impl Builder for HfBuilder {
debug!("backend use root: {}", &root);
debug!("backend use token: {}", token.is_some());
debug!("backend use endpoint: {}", &endpoint);
- let download_mode = self.config.download_mode.unwrap_or_default();
debug!("backend use download_mode: {:?}", download_mode);
let info = ServiceInfo::new(HF_SCHEME, "", "");
@@ -541,6 +563,40 @@ mod tests {
assert_eq!(result, "https://env.example.com";);
}
+ #[test]
+ fn hf_download_mode_defaults_to_xet() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_XET") };
+ assert_eq!(HfBuilder::default().hf_download_mode(),
HfDownloadMode::Xet);
+ }
+
+ #[test]
+ fn hf_download_mode_disable_xet_env_forces_http() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_HUB_DISABLE_XET", "1") };
+ let mode = HfBuilder::default().hf_download_mode();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_XET") };
+ assert_eq!(mode, HfDownloadMode::Http);
+ }
+
+ #[test]
+ fn hf_download_mode_config_takes_priority_over_env() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_HUB_DISABLE_XET", "1") };
+ let mode =
HfBuilder::default().download_mode("xet").hf_download_mode();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_XET") };
+ assert_eq!(mode, HfDownloadMode::Xet);
+ }
+
+ #[test]
+ fn hf_download_mode_empty_env_keeps_xet() {
+ let _guard = ENV_LOCK.lock().unwrap();
+ unsafe { std::env::set_var("HF_HUB_DISABLE_XET", "") };
+ let mode = HfBuilder::default().hf_download_mode();
+ unsafe { std::env::remove_var("HF_HUB_DISABLE_XET") };
+ assert_eq!(mode, HfDownloadMode::Xet);
+ }
+
#[test]
fn build_accepts_datasets_alias() {
HfBuilder::default()
diff --git a/core/services/hf/src/config.rs b/core/services/hf/src/config.rs
index 938a56b42..5ef944531 100644
--- a/core/services/hf/src/config.rs
+++ b/core/services/hf/src/config.rs
@@ -53,6 +53,12 @@ pub struct HfConfig {
/// Default is "https://huggingface.co";.
pub endpoint: Option<String>,
/// Download mode. Either `xet` (default) or `http`.
+ ///
+ /// When unset, the mode is resolved from the `HF_HUB_DISABLE_XET`
+ /// environment variable: a non-empty value forces `http`, otherwise it
+ /// defaults to `xet`. An explicit value here takes precedence.
+ ///
+ /// See
<https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables#hfhubdisablexet>.
pub download_mode: Option<HfDownloadMode>,
}
diff --git a/core/services/hf/src/docs.md b/core/services/hf/src/docs.md
index c752efb52..bcb5ef710 100644
--- a/core/services/hf/src/docs.md
+++ b/core/services/hf/src/docs.md
@@ -31,6 +31,7 @@ This service can be used to:
- `root`: Set the work directory for backend.
- `token`: The token for accessing the repository. Required for write
operations.
- `endpoint`: The Hub base URL. Default is `https://huggingface.co`. Can also
be set via the `HF_ENDPOINT` environment variable.
+- `download_mode`: How files are downloaded. One of `xet` (default) or `http`.
When unset, a non-empty `HF_HUB_DISABLE_XET` environment variable forces `http`.
Refer to [`HfBuilder`]'s public API docs for more information.
