This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 1823a52e36 NIFI-13884 Removed File IO Implementation Property from
PutIceberg (#9403)
1823a52e36 is described below
commit 1823a52e3610fd207957365dfc41ddcfa9d777f1
Author: Mark Bathori <[email protected]>
AuthorDate: Thu Oct 17 16:00:36 2024 +0200
NIFI-13884 Removed File IO Implementation Property from PutIceberg (#9403)
- Removed pending further testing and evaluation of runtime behavior
Signed-off-by: David Handermann <[email protected]>
---
.../nifi-iceberg-common/pom.xml | 15 -----
.../iceberg/catalog/IcebergCatalogFactory.java | 3 +-
.../services/iceberg/IcebergCatalogProperty.java | 3 +-
.../services/iceberg/FileIOImplementation.java | 68 ----------------------
.../nifi/services/iceberg/JdbcCatalogService.java | 14 +----
5 files changed, 3 insertions(+), 100 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
index 7ef3aa0065..232068391b 100644
--- a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
+++ b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
@@ -89,21 +89,6 @@
<artifactId>iceberg-orc</artifactId>
<version>${iceberg.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-aws</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-azure</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.iceberg</groupId>
- <artifactId>iceberg-gcp</artifactId>
- <version>${iceberg.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
diff --git
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
index 19da644afb..eac1981735 100644
---
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
+++
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
@@ -36,7 +36,6 @@ import java.util.function.Function;
import static
org.apache.nifi.processors.iceberg.IcebergUtils.getConfigurationFromFiles;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.CATALOG_NAME;
-import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.METASTORE_URI;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION;
@@ -101,7 +100,7 @@ public class IcebergCatalogFactory {
final DBCPService dbcpService = (DBCPService)
catalogProperties.get(CLIENT_POOL_SERVICE);
final Function<Map<String, String>, JdbcClientPool> clientPoolBuilder
= props -> new IcebergJdbcClientPool(props, dbcpService);
- final Function<Map<String, String>, FileIO> ioBuilder = props ->
CatalogUtil.loadFileIO((String) catalogProperties.get(FILE_IO_IMPLEMENTATION),
props, configuration);
+ final Function<Map<String, String>, FileIO> ioBuilder = props ->
CatalogUtil.loadFileIO("org.apache.iceberg.hadoop.HadoopFileIO", props,
configuration);
JdbcCatalog catalog = new JdbcCatalog(ioBuilder, clientPoolBuilder,
false);
catalog.setConf(configuration);
diff --git
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java
index 1e4987a0a5..0874f731ad 100644
---
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java
+++
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services-api/src/main/java/org/apache/nifi/services/iceberg/IcebergCatalogProperty.java
@@ -22,8 +22,7 @@ public enum IcebergCatalogProperty {
CATALOG_NAME,
METASTORE_URI("hive.metastore.uris"),
WAREHOUSE_LOCATION("hive.metastore.warehouse.dir"),
- CLIENT_POOL_SERVICE,
- FILE_IO_IMPLEMENTATION;
+ CLIENT_POOL_SERVICE;
private static final String EMPTY_STRING = "";
diff --git
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java
deleted file mode 100644
index 45c3d3cfaa..0000000000
---
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/FileIOImplementation.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
-
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.services.iceberg;
-
-import org.apache.nifi.components.DescribedValue;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public enum FileIOImplementation implements DescribedValue {
- HADOOP( "org.apache.iceberg.hadoop.HadoopFileIO", "Hadoop File IO"),
- RESOLVING("org.apache.iceberg.io.ResolvingFileIO", "Resolving File IO"),
- S3( "org.apache.iceberg.aws.s3.S3FileIO", "S3 File IO"),
- GCS( "org.apache.iceberg.gcp.gcs.GCSFileIO", "GCS File IO"),
- ADLS( "org.apache.iceberg.azure.adlsv2.ADLSFileIO", "ADLS File IO");
-
- private static final Map<String, FileIOImplementation> ENUM_MAP = new
HashMap<>();
-
- static {
- for (FileIOImplementation strategy : FileIOImplementation.values()) {
- ENUM_MAP.put(strategy.getValue(), strategy);
- }
- }
-
- private final String value;
- private final String displayName;
- private final String description;
-
- FileIOImplementation(String value, String displayName) {
- this(value, displayName, null);
- }
-
- FileIOImplementation(String value, String displayName, String description)
{
- this.value = value;
- this.displayName = displayName;
- this.description = description;
- }
-
- @Override
- public String getValue() {
- return value;
- }
-
- @Override
- public String getDisplayName() {
- return displayName;
- }
-
- @Override
- public String getDescription() {
- return description;
- }
-}
diff --git
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java
index 9a596aa06f..7f8be77e06 100644
---
a/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java
+++
b/nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-services/src/main/java/org/apache/nifi/services/iceberg/JdbcCatalogService.java
@@ -28,8 +28,6 @@ import org.apache.nifi.processor.util.StandardValidators;
import java.util.List;
-import static org.apache.nifi.services.iceberg.FileIOImplementation.HADOOP;
-import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.FILE_IO_IMPLEMENTATION;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE;
import static
org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION;
@@ -53,17 +51,8 @@ public class JdbcCatalogService extends
AbstractCatalogService {
.required(true)
.build();
- public static final PropertyDescriptor FILE_IO_IMPL = new
PropertyDescriptor.Builder()
- .name("File IO Implementation")
- .description("Specifies the implementation of FileIO interface to
be used. " +
- "The provided implementation have to include the class and
full package name.")
- .required(true)
- .defaultValue(HADOOP.getValue())
- .allowableValues(FileIOImplementation.class)
- .build();
-
private static final List<PropertyDescriptor> PROPERTIES = List.of(
- CATALOG_NAME, CONNECTION_POOL, FILE_IO_IMPL, WAREHOUSE_PATH,
HADOOP_CONFIGURATION_RESOURCES);
+ CATALOG_NAME, CONNECTION_POOL, WAREHOUSE_PATH,
HADOOP_CONFIGURATION_RESOURCES);
@Override
protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
@@ -78,7 +67,6 @@ public class JdbcCatalogService extends
AbstractCatalogService {
catalogProperties.put(IcebergCatalogProperty.CATALOG_NAME,
context.getProperty(CATALOG_NAME).evaluateAttributeExpressions().getValue());
catalogProperties.put(CLIENT_POOL_SERVICE,
context.getProperty(CONNECTION_POOL).asControllerService(DBCPService.class));
- catalogProperties.put(FILE_IO_IMPLEMENTATION,
context.getProperty(FILE_IO_IMPL).evaluateAttributeExpressions().getValue());
catalogProperties.put(WAREHOUSE_LOCATION,
context.getProperty(WAREHOUSE_PATH).evaluateAttributeExpressions().getValue());
}
