This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new a0c9a3248e [#9765] improvement(iceberg-rest-server): Improve
performance of loading tables (#9766)
a0c9a3248e is described below
commit a0c9a3248e6105de1d9f745ec7a2865a123a5ae8
Author: roryqi <[email protected]>
AuthorDate: Tue Jan 27 21:05:37 2026 +0800
[#9765] improvement(iceberg-rest-server): Improve performance of loading
tables (#9766)
### What changes were proposed in this pull request?
Improve the performance of loading tables
### Why are the changes needed?
This improvement reuse the method in
https://github.com/apache/gravitino/pull/6576
Fix #9765
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Passed performance tests.
---
.../gravitino/oss/fs/OSSFileSystemProvider.java | 2 +-
.../apache/gravitino/s3/fs/S3FileSystemProvider.java | 2 +-
.../gravitino/abs/fs/AzureFileSystemProvider.java | 2 +-
.../gravitino/gcs/fs/GCSFileSystemProvider.java | 2 +-
.../fileset/TestFilesetCatalogOperations.java | 3 ++-
.../catalog/fileset/fs/TestFileSystemUtils.java | 2 +-
.../gravitino/catalog/hadoop/fs/FileSystemUtils.java | 20 +++++++++++++++-----
.../catalog/hadoop/fs/HDFSFileSystemProvider.java | 2 +-
.../catalog/hadoop/fs/HDFSFileSystemProxy.java | 3 ++-
.../catalog/hadoop/fs/LocalFileSystemProvider.java | 3 ++-
iceberg/iceberg-common/build.gradle.kts | 3 +++
.../iceberg/common/ops/IcebergCatalogWrapper.java | 5 +++--
12 files changed, 33 insertions(+), 16 deletions(-)
diff --git
a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
index 73196cd354..0f443dbcf2 100644
---
a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
+++
b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
@@ -70,7 +70,7 @@ public class OSSFileSystemProvider implements
FileSystemProvider, SupportsCreden
hadoopConfMap = additionalOSSConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
return AliyunOSSFileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
index 4e04216ac7..4e4d6152d2 100644
---
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
+++
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
@@ -82,7 +82,7 @@ public class S3FileSystemProvider implements
FileSystemProvider, SupportsCredent
hadoopConfMap = additionalS3Config(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
return S3AFileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
index 4a107e0bd6..08d780c09f 100644
---
a/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
+++
b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
@@ -75,7 +75,7 @@ public class AzureFileSystemProvider implements
FileSystemProvider, SupportsCred
hadoopConfMap = additionalAzureConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
index 41cc0047d6..ac62d43331 100644
---
a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
+++
b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
@@ -54,7 +54,7 @@ public class GCSFileSystemProvider implements
FileSystemProvider, SupportsCreden
FileSystemUtils.toHadoopConfigMap(config,
GRAVITINO_KEY_TO_GCS_HADOOP_KEY);
hadoopConfMap = additionalGCSConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
index 6d59d261a9..a0dbce64a2 100644
---
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
+++
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
@@ -358,7 +358,8 @@ public class TestFilesetCatalogOperations {
CatalogInfo catalogInfo = randomCatalogInfo();
ops.initialize(emptyProps, catalogInfo, FILESET_PROPERTIES_METADATA);
- Configuration conf = FileSystemUtils.createConfiguration(GRAVITINO_BYPASS,
ops.getConf());
+ Configuration conf =
+ FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
ops.getConf());
String value = conf.get("fs.defaultFS");
Assertions.assertEquals("file:///", value);
diff --git
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
index 769e5e25ad..1dbc80d8b4 100644
---
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
+++
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
@@ -49,7 +49,7 @@ public class TestFileSystemUtils {
"fs.s3a.impl", "v2",
"fs.s3a.endpoint", "v3",
"gravitino.bypass.fs.s3a.endpoint", "v4");
- Configuration configuration = FileSystemUtils.createConfiguration(confMap);
+ Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(confMap);
Assertions.assertEquals("v1", configuration.get("s3a-endpoint"));
Assertions.assertEquals("v2", configuration.get("fs.s3a.impl"));
Assertions.assertEquals("v3", configuration.get("fs.s3a.endpoint"));
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
index c2a789ea74..f9d38e6975 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
@@ -168,8 +168,11 @@ public class FileSystemUtils {
* @param config properties map
* @return Configuration map
*/
- public static Configuration createConfiguration(Map<String, String> config) {
- return createConfiguration(null, config);
+ public static Configuration createCompatibleConfiguration(Map<String,
String> config) {
+ Map<String, String> updatedConfig = Maps.newHashMap(config);
+ updatedConfig.put(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, "true");
+ updatedConfig.put(FS_DISABLE_CACHE, "true");
+ return createCompatibleConfiguration(null, updatedConfig);
}
/**
@@ -179,9 +182,19 @@ public class FileSystemUtils {
* @param config properties map
* @return Configuration map
*/
+ public static Configuration createCompatibleConfiguration(
+ String bypass, Map<String, String> config) {
+ Map<String, String> updatedConfig = Maps.newHashMap(config);
+ updatedConfig.put(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, "true");
+ updatedConfig.put(FS_DISABLE_CACHE, "true");
+ return createConfiguration(bypass, updatedConfig);
+ }
+
public static Configuration createConfiguration(String bypass, Map<String,
String> config) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
Configuration configuration = new Configuration();
+ // Please don't call the method configuration.set(key, value) here,
because it will
+ // cause performance issue.
String hdfsConfigResources = config.get(CONFIG_RESOURCES);
if (StringUtils.isNotBlank(hdfsConfigResources)) {
@@ -205,9 +218,6 @@ public class FileSystemUtils {
writer.close();
configuration.addResource(new ByteArrayInputStream(out.toByteArray()));
- configuration.setBoolean(FS_DISABLE_CACHE, true);
- configuration.setBoolean(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, true);
-
return configuration;
} catch (Exception e) {
throw new RuntimeException("Failed to create configuration", e);
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
index e9a087558d..0711b008be 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
@@ -46,7 +46,7 @@ public class HDFSFileSystemProvider implements
FileSystemProvider {
throws IOException {
Map<String, String> hadoopConfMap = additionalHDFSConfig(config);
Configuration configuration =
- FileSystemUtils.createConfiguration(GRAVITINO_BYPASS, hadoopConfMap);
+ FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProxy.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProxy.java
index 902e5f3f49..cf27d04903 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProxy.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProxy.java
@@ -70,7 +70,8 @@ public class HDFSFileSystemProxy implements MethodInterceptor
{
protected void initFileSystem(Path path, Map<String, String> config) {
try {
Map<String, String> hadoopConfMap = additionalHDFSConfig(config);
- Configuration conf =
FileSystemUtils.createConfiguration(GRAVITINO_BYPASS, hadoopConfMap);
+ Configuration conf =
+ FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
hadoopConfMap);
conf.setBoolean(FS_DISABLE_CACHE, true);
conf.setBoolean(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, true);
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
index d0d34f16ec..2460e7f19c 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
@@ -30,7 +30,8 @@ public class LocalFileSystemProvider implements
FileSystemProvider {
@Override
public FileSystem getFileSystem(Path path, Map<String, String> config)
throws IOException {
- Configuration configuration =
FileSystemUtils.createConfiguration(GRAVITINO_BYPASS, config);
+ Configuration configuration =
+ FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
config);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git a/iceberg/iceberg-common/build.gradle.kts
b/iceberg/iceberg-common/build.gradle.kts
index 50c052d59c..16d1963a4b 100644
--- a/iceberg/iceberg-common/build.gradle.kts
+++ b/iceberg/iceberg-common/build.gradle.kts
@@ -27,6 +27,9 @@ plugins {
dependencies {
implementation(project(":api"))
implementation(project(":catalogs:catalog-common"))
+ implementation(project(":catalogs:hadoop-common")) {
+ exclude("*")
+ }
implementation(project(":core")) {
exclude("*")
}
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
index 04c0627dfb..864e538649 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
@@ -27,6 +27,7 @@ import java.util.Optional;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
+import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.iceberg.common.cache.SupportsMetadataLocation;
@@ -77,6 +78,7 @@ public class IcebergCatalogWrapper implements AutoCloseable {
private String catalogUri = null;
private Map<String, String> catalogPropertiesMap;
private TableMetadataCache metadataCache;
+ private Configuration configuration;
public IcebergCatalogWrapper(IcebergConfig icebergConfig) {
this.icebergConfig = icebergConfig;
@@ -100,6 +102,7 @@ public class IcebergCatalogWrapper implements AutoCloseable
{
this.metadataCache = loadTableMetadataCache(icebergConfig, catalog);
this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties();
+ this.configuration = FileSystemUtils.createConfiguration(null,
catalogPropertiesMap);
}
private void validateNamespace(Optional<Namespace> namespace) {
@@ -162,8 +165,6 @@ public class IcebergCatalogWrapper implements AutoCloseable
{
* reinitialize it again.
*/
public void reloadHadoopConf() {
- Configuration configuration = new Configuration();
- this.catalogPropertiesMap.forEach(configuration::set);
UserGroupInformation.setConfiguration(configuration);
}
