This is an automated email from the ASF dual-hosted git repository.
mosermw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 4d0696afb7 NIFI-14343 Replaced IdentifyMimeType property descriptors
Config File and Config Body with one property descriptor using
identifiesExternalResource.
4d0696afb7 is described below
commit 4d0696afb76d66b873bb33a54d4798530d978781
Author: dan-s1 <[email protected]>
AuthorDate: Mon Mar 10 21:28:33 2025 +0000
NIFI-14343 Replaced IdentifyMimeType property descriptors Config File and
Config Body with one property descriptor using identifiesExternalResource.
Signed-off-by: Mike Moser <[email protected]>
Closes #9793
---
.../nifi/processors/standard/IdentifyMimeType.java | 118 ++++++++++-----------
.../processors/standard/TestIdentifyMimeType.java | 34 +++---
2 files changed, 75 insertions(+), 77 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
index b6efd68a07..da465ee144 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/IdentifyMimeType.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.processors.standard;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
import org.apache.nifi.annotation.behavior.SideEffectFree;
@@ -29,8 +30,9 @@ import org.apache.nifi.components.AllowableValue;
import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
-import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.components.resource.ResourceCardinality;
+import org.apache.nifi.components.resource.ResourceType;
+import org.apache.nifi.context.PropertyContext;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.logging.ComponentLog;
@@ -54,8 +56,6 @@ import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
import org.apache.tika.mime.MimeTypesFactory;
-import java.io.ByteArrayInputStream;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.Charset;
@@ -90,8 +90,8 @@ import java.util.Set;
)
public class IdentifyMimeType extends AbstractProcessor {
static final AllowableValue PRESET = new AllowableValue("Preset",
"Preset", "Use default NiFi MIME Types.");
- static final AllowableValue REPLACE = new AllowableValue("Replace",
"Replace", "Use config MIME Types only.");
- static final AllowableValue MERGE = new AllowableValue("Merge", "Merge",
"Use config together with default NiFi MIME Types.");
+ static final AllowableValue REPLACE = new AllowableValue("Replace",
"Replace", "Use custom MIME types configuration only.");
+ static final AllowableValue MERGE = new AllowableValue("Merge", "Merge",
"Use custom MIME types configuration together with default NiFi MIME types.");
public static final PropertyDescriptor USE_FILENAME_IN_DETECTION = new
PropertyDescriptor.Builder()
.displayName("Use Filename In Detection")
@@ -111,31 +111,19 @@ public class IdentifyMimeType extends AbstractProcessor {
.defaultValue(PRESET.getValue())
.build();
- public static final PropertyDescriptor MIME_CONFIG_FILE = new
PropertyDescriptor.Builder()
- .displayName("Config File")
- .name("config-file")
- .required(false)
- .description("Path to MIME type config file. Only one of Config
File or Config Body may be used.")
- .addValidator(new StandardValidators.FileExistsValidator(true))
- .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT)
- .dependsOn(CONFIG_STRATEGY, REPLACE, MERGE)
- .build();
-
- public static final PropertyDescriptor MIME_CONFIG_BODY = new
PropertyDescriptor.Builder()
- .displayName("Config Body")
- .name("config-body")
- .required(false)
- .description("Body of MIME type config file. Only one of Config
File or Config Body may be used.")
- .addValidator(Validator.VALID)
- .expressionLanguageSupported(ExpressionLanguageScope.NONE)
+ public static final PropertyDescriptor CUSTOM_MIME_CONFIGURATION = new
PropertyDescriptor.Builder()
+ .name("Custom MIME Configuration")
+ .description("A URL or file path to a custom Tika Mime type
configuration or the actual content of a custom Tika Mime type configuration.")
+ .required(true)
+ .identifiesExternalResource(ResourceCardinality.SINGLE,
ResourceType.FILE, ResourceType.URL, ResourceType.TEXT)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.dependsOn(CONFIG_STRATEGY, REPLACE, MERGE)
.build();
private static final List<PropertyDescriptor> PROPERTY_DESCRIPTORS =
List.of(
USE_FILENAME_IN_DETECTION,
CONFIG_STRATEGY,
- MIME_CONFIG_BODY,
- MIME_CONFIG_FILE
+ CUSTOM_MIME_CONFIGURATION
);
public static final Relationship REL_SUCCESS = new Relationship.Builder()
@@ -147,6 +135,8 @@ public class IdentifyMimeType extends AbstractProcessor {
REL_SUCCESS
);
+ private static final String CUSTOM_MIME_TYPES_FILENAME =
"custom-mimetypes.xml";
+ private static final String DEFAULT_MIME_TYPES_PATH =
"org/apache/tika/mime/tika-mimetypes.xml";
private final TikaConfig config;
private Detector detector;
private EncodingDetector encodingDetector;
@@ -158,11 +148,23 @@ public class IdentifyMimeType extends AbstractProcessor {
@Override
public void migrateProperties(PropertyConfiguration config) {
+ final String configFileProperty = "config-file";
+ final String configBodyProperty = "config-body";
+
if (!config.hasProperty(CONFIG_STRATEGY)) {
- if (config.isPropertySet(MIME_CONFIG_FILE) ||
config.isPropertySet(MIME_CONFIG_BODY)) {
+ if (config.isPropertySet(configFileProperty) ||
config.isPropertySet(configBodyProperty)) {
config.setProperty(CONFIG_STRATEGY, REPLACE.getValue());
}
}
+
+ if (config.isPropertySet(configFileProperty)) {
+ config.setProperty(CUSTOM_MIME_CONFIGURATION.getName(),
config.getRawPropertyValue(configFileProperty).orElse(""));
+ } else if (config.isPropertySet(configBodyProperty)) {
+ config.setProperty(CUSTOM_MIME_CONFIGURATION.getName(),
config.getRawPropertyValue(configBodyProperty).orElse(""));
+ }
+
+ config.removeProperty(configFileProperty);
+ config.removeProperty(configBodyProperty);
}
@OnScheduled
@@ -173,33 +175,18 @@ public class IdentifyMimeType extends AbstractProcessor {
this.detector = config.getDetector();
this.mimeTypes = config.getMimeRepository();
} else {
- setCustomMimeTypes(configStrategy, context);
+ try {
+ this.detector = createCustomMimeTypes(configStrategy, context);
+ this.mimeTypes = (MimeTypes) this.detector;
+ } catch (Exception e) {
+ context.yield();
+ throw new ProcessException("Failed to load configuration", e);
+ }
}
this.encodingDetector = config.getEncodingDetector();
}
- private void setCustomMimeTypes(String configStrategy, ProcessContext
context) {
- String configBody = context.getProperty(MIME_CONFIG_BODY).getValue();
- String configFile =
context.getProperty(MIME_CONFIG_FILE).evaluateAttributeExpressions().getValue();
-
- try (final InputStream customInputStream = configBody != null ? new
ByteArrayInputStream(configBody.getBytes()) : new FileInputStream(configFile)) {
- if (configStrategy.equals(REPLACE.getValue())) {
- this.detector = MimeTypesFactory.create(customInputStream);
- } else {
- try (final InputStream nifiInputStream =
getClass().getClassLoader().getResourceAsStream("custom-mimetypes.xml");
- final InputStream tikaInputStream =
MimeTypes.class.getClassLoader().getResourceAsStream("org/apache/tika/mime/tika-mimetypes.xml"))
{
- this.detector = MimeTypesFactory.create(customInputStream,
nifiInputStream, tikaInputStream);
- }
- }
- this.mimeTypes = (MimeTypes) this.detector;
- } catch (Exception e) {
- context.yield();
- String configSource = configBody != null ? "body" : "file";
- throw new ProcessException("Failed to load config " +
configSource, e);
- }
- }
-
@Override
public Set<Relationship> getRelationships() {
return RELATIONSHIPS;
@@ -277,29 +264,36 @@ public class IdentifyMimeType extends AbstractProcessor {
return null;
}
}
-
@Override
protected Collection<ValidationResult> customValidate(ValidationContext
validationContext) {
Set<ValidationResult> results = new HashSet<>();
- String body =
validationContext.getProperty(MIME_CONFIG_BODY).getValue();
- String file =
validationContext.getProperty(MIME_CONFIG_FILE).getValue();
- if
(!validationContext.getProperty(CONFIG_STRATEGY).getValue().equals(PRESET.getValue()))
{
- if (body != null && file != null) {
- results.add(new ValidationResult.Builder()
- .subject(MIME_CONFIG_FILE.getDisplayName())
- .input(file)
- .valid(false)
- .explanation("Either [Config Body] or [Config File]
can be specified, but not both properties.")
- .build());
- } else if (body == null && file == null) {
+ String configStrategy =
validationContext.getProperty(CONFIG_STRATEGY).getValue();
+
+ if (!configStrategy.equals(PRESET.getValue())) {
+ try {
+ createCustomMimeTypes(configStrategy, validationContext);
+ } catch (Exception e) {
results.add(new ValidationResult.Builder()
- .subject(MIME_CONFIG_FILE.getDisplayName())
+ .subject(CUSTOM_MIME_CONFIGURATION.getDisplayName())
+
.input(validationContext.getProperty(CUSTOM_MIME_CONFIGURATION).getValue())
.valid(false)
- .explanation("Either [Config Body] or [Config File]
must be specified")
+ .explanation("Invalid configuration " +
ExceptionUtils.getRootCauseMessage(e))
.build());
}
}
return results;
}
+ private MimeTypes createCustomMimeTypes(String configStrategy,
PropertyContext context) throws MimeTypeException, IOException {
+ try (final InputStream customInputStream =
context.getProperty(CUSTOM_MIME_CONFIGURATION).asResource().read()) {
+ if (configStrategy.equals(REPLACE.getValue())) {
+ return MimeTypesFactory.create(customInputStream);
+ } else {
+ try (final InputStream nifiInputStream =
getClass().getClassLoader().getResourceAsStream(CUSTOM_MIME_TYPES_FILENAME);
+ final InputStream tikaInputStream =
MimeTypes.class.getClassLoader().getResourceAsStream(DEFAULT_MIME_TYPES_PATH)) {
+ return MimeTypesFactory.create(customInputStream,
nifiInputStream, tikaInputStream);
+ }
+ }
+ }
+ }
}
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
index 69c38b60e1..ffcc5a996a 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestIdentifyMimeType.java
@@ -19,6 +19,7 @@ package org.apache.nifi.processors.standard;
import org.apache.nifi.flowfile.attributes.CoreAttributes;
import org.apache.nifi.flowfile.attributes.StandardFlowFileMediaType;
import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.PropertyMigrationResult;
import org.apache.nifi.util.TestRunner;
import org.apache.nifi.util.TestRunners;
import org.junit.jupiter.api.BeforeEach;
@@ -33,11 +34,11 @@ import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertThrows;
class TestIdentifyMimeType {
@@ -138,7 +139,7 @@ class TestIdentifyMimeType {
@MethodSource("replaceWithConfigBodyData")
void testReplaceWithConfigBody(File file, Map<String, String>
expectedMimeTypes, Map<String, String> expectedExtensions) throws IOException {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.REPLACE);
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_BODY, CONFIG_BODY);
+ runner.setProperty(IdentifyMimeType.CUSTOM_MIME_CONFIGURATION,
CONFIG_BODY);
runner.enqueue(file.toPath());
runner.run();
@@ -213,7 +214,7 @@ class TestIdentifyMimeType {
@MethodSource("replaceWithConfigFileData")
void testReplaceWithConfigFile(File file, Map<String, String>
expectedMimeTypes, Map<String, String> expectedExtensions) throws IOException {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.REPLACE);
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_FILE, CONFIG_FILE);
+ runner.setProperty(IdentifyMimeType.CUSTOM_MIME_CONFIGURATION,
CONFIG_FILE);
runner.enqueue(file.toPath());
runner.run();
@@ -287,7 +288,7 @@ class TestIdentifyMimeType {
@MethodSource("mergeWithConfigBodyData")
void testMergeWithConfigBody(File file, Map<String, String>
expectedMimeTypes, Map<String, String> expectedExtensions) throws IOException {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.MERGE);
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_BODY, CONFIG_BODY);
+ runner.setProperty(IdentifyMimeType.CUSTOM_MIME_CONFIGURATION,
CONFIG_BODY);
runner.enqueue(file.toPath());
runner.run();
@@ -325,7 +326,7 @@ class TestIdentifyMimeType {
@MethodSource("mergeWithConfigFileData")
void testMergeWithConfigFile(File file, Map<String, String>
expectedMimeTypes, Map<String, String> expectedExtensions) throws IOException {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.MERGE);
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_FILE, CONFIG_FILE);
+ runner.setProperty(IdentifyMimeType.CUSTOM_MIME_CONFIGURATION,
CONFIG_FILE);
runner.enqueue(file.toPath());
runner.run();
@@ -361,23 +362,26 @@ class TestIdentifyMimeType {
}
@Test
- void testOnlyOneCustomMimeConfigSpecified() {
+ void testNoReplaceCustomMimeConfigurationSpecified() {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.REPLACE);
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_FILE, CONFIG_FILE);
- String configBody = "foo";
- runner.setProperty(IdentifyMimeType.MIME_CONFIG_BODY, configBody);
-
- runner.setThreadCount(1);
- assertThrows(AssertionError.class, () -> runner.run());
+ runner.assertNotValid();
}
@Test
- void testNoCustomMimeConfigSpecified() {
+ void testInvalidCustomConfigurationReplace() {
runner.setProperty(IdentifyMimeType.CONFIG_STRATEGY,
IdentifyMimeType.REPLACE);
+ runner.setProperty(IdentifyMimeType.CUSTOM_MIME_CONFIGURATION,
"gibberish");
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ void testMigration() {
+ final PropertyMigrationResult propertyMigrationResult =
runner.migrateProperties();
+ final Set<String> expectedPropertiesRemoved = Set.of("config-file",
"config-body");
- runner.setThreadCount(1);
- assertThrows(AssertionError.class, () -> runner.run());
+ assertEquals(expectedPropertiesRemoved,
propertyMigrationResult.getPropertiesRemoved());
}
private static Map<String, String> getCommonExpectedMimeTypes() {
