This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch branch-1.1
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-1.1 by this push:
new fc86e22c63 Revert "[#9765] improvement(iceberg-rest-server): Improve
performance of loading tables (#9766)"
fc86e22c63 is described below
commit fc86e22c63f1e989ea1c2433cbf3de8c6324bfc5
Author: Rory <[email protected]>
AuthorDate: Tue Jan 27 21:11:54 2026 +0800
Revert "[#9765] improvement(iceberg-rest-server): Improve performance of
loading tables (#9766)"
This reverts commit c6bcb7ed0af7e89d6641044a63eab8952f2eda02.
---
.../gravitino/oss/fs/OSSFileSystemProvider.java | 2 +-
.../apache/gravitino/s3/fs/S3FileSystemProvider.java | 2 +-
.../gravitino/abs/fs/AzureFileSystemProvider.java | 2 +-
.../gravitino/gcs/fs/GCSFileSystemProvider.java | 2 +-
.../fileset/TestFilesetCatalogOperations.java | 3 +--
.../catalog/fileset/fs/TestFileSystemUtils.java | 2 +-
.../gravitino/catalog/hadoop/fs/FileSystemUtils.java | 20 +++++---------------
.../catalog/hadoop/fs/HDFSFileSystemProvider.java | 2 +-
.../catalog/hadoop/fs/LocalFileSystemProvider.java | 3 +--
iceberg/iceberg-common/build.gradle.kts | 3 ---
.../iceberg/common/ops/IcebergCatalogWrapper.java | 5 ++---
11 files changed, 15 insertions(+), 31 deletions(-)
diff --git
a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
index 0f443dbcf2..73196cd354 100644
---
a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
+++
b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java
@@ -70,7 +70,7 @@ public class OSSFileSystemProvider implements
FileSystemProvider, SupportsCreden
hadoopConfMap = additionalOSSConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
return AliyunOSSFileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
index 4e4d6152d2..4e04216ac7 100644
---
a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
+++
b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
@@ -82,7 +82,7 @@ public class S3FileSystemProvider implements
FileSystemProvider, SupportsCredent
hadoopConfMap = additionalS3Config(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
return S3AFileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
index 08d780c09f..4a107e0bd6 100644
---
a/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
+++
b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
@@ -75,7 +75,7 @@ public class AzureFileSystemProvider implements
FileSystemProvider, SupportsCred
hadoopConfMap = additionalAzureConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
index ac62d43331..41cc0047d6 100644
---
a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
+++
b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
@@ -54,7 +54,7 @@ public class GCSFileSystemProvider implements
FileSystemProvider, SupportsCreden
FileSystemUtils.toHadoopConfigMap(config,
GRAVITINO_KEY_TO_GCS_HADOOP_KEY);
hadoopConfMap = additionalGCSConfig(hadoopConfMap);
- Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(hadoopConfMap);
+ Configuration configuration =
FileSystemUtils.createConfiguration(hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
index 3f6ba55b52..a55b519f62 100644
---
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
+++
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCatalogOperations.java
@@ -358,8 +358,7 @@ public class TestFilesetCatalogOperations {
CatalogInfo catalogInfo = randomCatalogInfo();
ops.initialize(emptyProps, catalogInfo, FILESET_PROPERTIES_METADATA);
- Configuration conf =
- FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
ops.getConf());
+ Configuration conf = FileSystemUtils.createConfiguration(GRAVITINO_BYPASS,
ops.getConf());
String value = conf.get("fs.defaultFS");
Assertions.assertEquals("file:///", value);
diff --git
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
index 1dbc80d8b4..769e5e25ad 100644
---
a/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
+++
b/catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/fs/TestFileSystemUtils.java
@@ -49,7 +49,7 @@ public class TestFileSystemUtils {
"fs.s3a.impl", "v2",
"fs.s3a.endpoint", "v3",
"gravitino.bypass.fs.s3a.endpoint", "v4");
- Configuration configuration =
FileSystemUtils.createCompatibleConfiguration(confMap);
+ Configuration configuration = FileSystemUtils.createConfiguration(confMap);
Assertions.assertEquals("v1", configuration.get("s3a-endpoint"));
Assertions.assertEquals("v2", configuration.get("fs.s3a.impl"));
Assertions.assertEquals("v3", configuration.get("fs.s3a.endpoint"));
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
index 5c82265df8..fac32500ce 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java
@@ -206,11 +206,8 @@ public class FileSystemUtils {
* @param config properties map
* @return Configuration map
*/
- public static Configuration createCompatibleConfiguration(Map<String,
String> config) {
- Map<String, String> updatedConfig = Maps.newHashMap(config);
- updatedConfig.put(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, "true");
- updatedConfig.put(FS_DISABLE_CACHE, "true");
- return createCompatibleConfiguration(null, updatedConfig);
+ public static Configuration createConfiguration(Map<String, String> config) {
+ return createConfiguration(null, config);
}
/**
@@ -220,19 +217,9 @@ public class FileSystemUtils {
* @param config properties map
* @return Configuration map
*/
- public static Configuration createCompatibleConfiguration(
- String bypass, Map<String, String> config) {
- Map<String, String> updatedConfig = Maps.newHashMap(config);
- updatedConfig.put(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, "true");
- updatedConfig.put(FS_DISABLE_CACHE, "true");
- return createConfiguration(bypass, updatedConfig);
- }
-
public static Configuration createConfiguration(String bypass, Map<String,
String> config) {
try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
Configuration configuration = new Configuration();
- // Please don't call the method configuration.set(key, value) here,
because it will
- // cause performance issue.
String hdfsConfigResources = config.get(CONFIG_RESOURCES);
if (StringUtils.isNotBlank(hdfsConfigResources)) {
@@ -256,6 +243,9 @@ public class FileSystemUtils {
writer.close();
configuration.addResource(new ByteArrayInputStream(out.toByteArray()));
+ configuration.setBoolean(FS_DISABLE_CACHE, true);
+ configuration.setBoolean(IPC_FALLBACK_TO_SIMPLE_AUTH_ALLOWED, true);
+
return configuration;
} catch (Exception e) {
throw new RuntimeException("Failed to create configuration", e);
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
index 6f600efc4d..27fc90d80a 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/HDFSFileSystemProvider.java
@@ -43,7 +43,7 @@ public class HDFSFileSystemProvider implements
FileSystemProvider {
throws IOException {
Map<String, String> hadoopConfMap = additionalHDFSConfig(config);
Configuration configuration =
- FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
hadoopConfMap);
+ FileSystemUtils.createConfiguration(GRAVITINO_BYPASS, hadoopConfMap);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
index 2460e7f19c..d0d34f16ec 100644
---
a/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
+++
b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/LocalFileSystemProvider.java
@@ -30,8 +30,7 @@ public class LocalFileSystemProvider implements
FileSystemProvider {
@Override
public FileSystem getFileSystem(Path path, Map<String, String> config)
throws IOException {
- Configuration configuration =
- FileSystemUtils.createCompatibleConfiguration(GRAVITINO_BYPASS,
config);
+ Configuration configuration =
FileSystemUtils.createConfiguration(GRAVITINO_BYPASS, config);
return FileSystem.newInstance(path.toUri(), configuration);
}
diff --git a/iceberg/iceberg-common/build.gradle.kts
b/iceberg/iceberg-common/build.gradle.kts
index 16d1963a4b..50c052d59c 100644
--- a/iceberg/iceberg-common/build.gradle.kts
+++ b/iceberg/iceberg-common/build.gradle.kts
@@ -27,9 +27,6 @@ plugins {
dependencies {
implementation(project(":api"))
implementation(project(":catalogs:catalog-common"))
- implementation(project(":catalogs:hadoop-common")) {
- exclude("*")
- }
implementation(project(":core")) {
exclude("*")
}
diff --git
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
index 864e538649..04c0627dfb 100644
---
a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
+++
b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java
@@ -27,7 +27,6 @@ import java.util.Optional;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
-import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergCatalogBackend;
import org.apache.gravitino.iceberg.common.IcebergConfig;
import org.apache.gravitino.iceberg.common.cache.SupportsMetadataLocation;
@@ -78,7 +77,6 @@ public class IcebergCatalogWrapper implements AutoCloseable {
private String catalogUri = null;
private Map<String, String> catalogPropertiesMap;
private TableMetadataCache metadataCache;
- private Configuration configuration;
public IcebergCatalogWrapper(IcebergConfig icebergConfig) {
this.icebergConfig = icebergConfig;
@@ -102,7 +100,6 @@ public class IcebergCatalogWrapper implements AutoCloseable
{
this.metadataCache = loadTableMetadataCache(icebergConfig, catalog);
this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties();
- this.configuration = FileSystemUtils.createConfiguration(null,
catalogPropertiesMap);
}
private void validateNamespace(Optional<Namespace> namespace) {
@@ -165,6 +162,8 @@ public class IcebergCatalogWrapper implements AutoCloseable
{
* reinitialize it again.
*/
public void reloadHadoopConf() {
+ Configuration configuration = new Configuration();
+ this.catalogPropertiesMap.forEach(configuration::set);
UserGroupInformation.setConfiguration(configuration);
}
