This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 541ccaa1bb3 HIVE-28575: Optimize FS rpc call in Warehouse::getDnsPath
(Butao Zhang, reviewed by Ayush Saxena, Chris Nauroth)
541ccaa1bb3 is described below
commit 541ccaa1bb35910b6af3036e4162d4bb952ea036
Author: Butao Zhang <[email protected]>
AuthorDate: Wed Mar 26 20:40:19 2025 +0800
HIVE-28575: Optimize FS rpc call in Warehouse::getDnsPath (Butao Zhang,
reviewed by Ayush Saxena, Chris Nauroth)
---
.../apache/hadoop/hive/metastore/Warehouse.java | 50 ++++++----------------
.../hadoop/hive/metastore/conf/MetastoreConf.java | 2 -
2 files changed, 14 insertions(+), 38 deletions(-)
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
index 2952276020c..57320d44892 100755
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java
@@ -133,14 +133,8 @@ public FileSystem getFs(Path f) throws MetaException {
/**
- * Hadoop File System reverse lookups paths with raw ip addresses The File
- * System URI always contains the canonical DNS name of the Namenode.
- * Subsequently, operations on paths with raw ip addresses cause an exception
- * since they don't match the file system URI.
- *
- * This routine solves this problem by replacing the scheme and authority of
a
- * path with the scheme and authority of the FileSystem that it maps to.
- *
+ * Get a path with canonical scheme and authority.
+ * <p>
* Since creating a new file system object is expensive, this method
* mimics getFileSystem() without creating an actual FileSystem object.
* When the input path lacks a scheme or an authority this is added
@@ -151,40 +145,24 @@ public FileSystem getFs(Path f) throws MetaException {
* @return Path with canonical scheme and authority
*/
public static Path getDnsPath(Path path, Configuration conf) throws
MetaException {
- if (isBlobStorageScheme(conf, path.toUri().getScheme())) {
- String scheme = path.toUri().getScheme();
- String authority = path.toUri().getAuthority();
- URI defaultUri = FileSystem.getDefaultUri(conf);
- if ((authority == null && scheme == null)
- || StringUtils.equalsIgnoreCase(scheme, defaultUri.getScheme()))
{
- if (authority == null) {
- authority = defaultUri.getAuthority();
- }
- if (scheme == null) {
- scheme = defaultUri.getScheme();
- }
- String uriPath = path.toUri().getPath();
- if (StringUtils.isEmpty(uriPath)) {
- uriPath = "/";
- }
- return new Path(scheme, authority, uriPath);
+ String scheme = path.toUri().getScheme();
+ String authority = path.toUri().getAuthority();
+ URI defaultUri = FileSystem.getDefaultUri(conf);
+ if ((authority == null && scheme == null)
+ || StringUtils.equalsIgnoreCase(scheme, defaultUri.getScheme())) {
+ if (authority == null) {
+ authority = defaultUri.getAuthority();
+ }
+ if (scheme == null) {
+ scheme = defaultUri.getScheme();
}
- return path;
- } else { // fallback: for other FS type make the FS instance
- FileSystem fs = getFs(path, conf);
String uriPath = path.toUri().getPath();
if (StringUtils.isEmpty(uriPath)) {
uriPath = "/";
}
- return (new Path(fs.getUri().getScheme(), fs.getUri().getAuthority(),
uriPath));
+ return new Path(scheme, authority, uriPath);
}
- }
-
- private static boolean isBlobStorageScheme(Configuration conf, String
scheme) {
- final String uriScheme = scheme == null ?
FileSystem.getDefaultUri(conf).getScheme() : scheme;
- return MetastoreConf.getStringCollection(conf,
MetastoreConf.ConfVars.HIVE_BLOBSTORE_SUPPORTED_SCHEMES)
- .stream()
- .anyMatch(each -> each.equalsIgnoreCase(uriScheme));
+ return path;
}
public Path getDnsPath(Path path) throws MetaException {
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index e0571664193..8a6d996818d 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1799,8 +1799,6 @@ public enum ConfVars {
"hive.metastore.custom.database.product.classname", "none",
"Hook for external RDBMS. This class will be instantiated only when
" +
"metastore.use.custom.database.product is set to true."),
- HIVE_BLOBSTORE_SUPPORTED_SCHEMES("hive.blobstore.supported.schemes",
"hive.blobstore.supported.schemes", "s3,s3a,s3n",
- "Comma-separated list of supported blobstore schemes."),
// Property-maps
PROPERTIES_CACHE_CAPACITY("hive.metastore.properties.cache.capacity",
