This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new b9101d2af4d branch-4.1: [fix](fe) Normalize default HDFS paths in
LocationPath #63476 (#63770)
b9101d2af4d is described below
commit b9101d2af4da21f56d7349cf2f0fe802865ebafb
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu May 28 17:15:46 2026 +0800
branch-4.1: [fix](fe) Normalize default HDFS paths in LocationPath #63476
(#63770)
Cherry-picked from #63476
Co-authored-by: Socrates <[email protected]>
---
.../java/org/apache/doris/common/util/LocationPath.java | 14 +++++++++++++-
.../org/apache/doris/common/util/LocationPathTest.java | 15 +++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
index c7558938711..452fc9b91a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
@@ -149,6 +149,9 @@ public class LocationPath {
String encodedLocation = encodedLocation(normalizedLocation);
URI uri = URI.create(encodedLocation);
String fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://" +
Strings.nullToEmpty(uri.getAuthority());
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
}
@@ -193,6 +196,9 @@ public class LocationPath {
URI uri = URI.create(encodedLocation);
String fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://"
+ Strings.nullToEmpty(uri.getAuthority());
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
} catch (UserException e) {
throw new StoragePropertiesException("Failed to create
LocationPath for location: " + location, e);
@@ -234,6 +240,7 @@ public class LocationPath {
String normalizedLocation =
storageProperties.validateAndNormalizeUri(location);
String fsIdentifier;
+ String schema = cachedSchema;
if (cachedFsIdPrefix != null &&
normalizedLocation.startsWith(cachedFsIdPrefix)) {
// Fast path: extract authority from normalized location
without full URI parsing
int authorityStart = cachedFsIdPrefix.length();
@@ -246,6 +253,9 @@ public class LocationPath {
throw new StoragePropertiesException("Invalid location,
missing authority: " + normalizedLocation);
}
fsIdentifier = cachedFsIdPrefix + authority;
+ if (StringUtils.isBlank(schema)) {
+ schema = cachedFsIdPrefix.substring(0,
cachedFsIdPrefix.length() - SCHEME_DELIM.length());
+ }
} else {
// Fallback to full URI parsing
String encodedLocation = encodedLocation(normalizedLocation);
@@ -256,9 +266,11 @@ public class LocationPath {
}
fsIdentifier = Strings.nullToEmpty(uri.getScheme()) + "://"
+ authority;
+ if (StringUtils.isBlank(schema)) {
+ schema = Strings.nullToEmpty(uri.getScheme());
+ }
}
- String schema = cachedSchema != null ? cachedSchema :
extractScheme(location);
return new LocationPath(schema, normalizedLocation, fsIdentifier,
storageProperties);
} catch (UserException e) {
throw new StoragePropertiesException("Failed to create
LocationPath for location: " + location, e);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
index 1e5d404a7f6..8f311225fcf 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/LocationPathTest.java
@@ -334,6 +334,21 @@ public class LocationPathTest {
Assertions.assertEquals(full.getSchema(), cached.getSchema());
}
+ @Test
+ public void testLocationPathWithCacheUsesDefaultFsForHdfsPath() {
+ StorageProperties storageProperties =
STORAGE_PROPERTIES_MAP.get(StorageProperties.Type.HDFS);
+ String location =
"/hadoop_catalog/fdm/f_csm_t_consume_info/data/data_dt=20220407/file.parquet";
+ LocationPath cached = LocationPath.ofWithCache(location,
storageProperties, null, null);
+ LocationPath full = LocationPath.of(location, STORAGE_PROPERTIES_MAP);
+ Assertions.assertEquals(full.getNormalizedLocation(),
cached.getNormalizedLocation());
+
Assertions.assertEquals("hdfs://namenode:8020/hadoop_catalog/fdm/f_csm_t_consume_info/data/"
+ + "data_dt=20220407/file.parquet",
cached.getNormalizedLocation());
+ Assertions.assertEquals("hdfs://namenode:8020",
cached.getFsIdentifier());
+ Assertions.assertEquals("hdfs", cached.getSchema());
+ Assertions.assertEquals(TFileType.FILE_HDFS,
cached.getTFileTypeForBE());
+ Assertions.assertEquals(FileSystemType.HDFS,
cached.getFileSystemType());
+ }
+
@Test
public void testLocationPathWithCacheMissingAuthority() {
StorageProperties storageProperties =
STORAGE_PROPERTIES_MAP.get(StorageProperties.Type.S3);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
