This is an automated email from the ASF dual-hosted git repository.
markap14 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new d2c70d1d2f NIFI-11434 added support for optionally allowing zip
entries with dat… (#7462)
d2c70d1d2f is described below
commit d2c70d1d2fdd8cf3e211676e0a6cc57d811cd2b2
Author: Joe Witt <[email protected]>
AuthorDate: Wed Jul 12 14:09:03 2023 -0700
NIFI-11434 added support for optionally allowing zip entries with dat…
(#7462)
* NIFI-11434 added support for optionally allowing zip entries with data
descriptors
* set default value and ensured all zip tests use various configs
---
.../nifi/processors/standard/UnpackContent.java | 34 ++++++++++++++++++----
.../processors/standard/TestUnpackContent.java | 8 ++++-
2 files changed, 36 insertions(+), 6 deletions(-)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
index 697b84ccd6..32f5e4df87 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/UnpackContent.java
@@ -162,6 +162,20 @@ public class UnpackContent extends AbstractProcessor {
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
.build();
+ public static final PropertyDescriptor
ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR = new PropertyDescriptor.Builder()
+ .name("allow-stored-entries-wdd")
+ .displayName("Allow Stored Entries With Data Descriptor")
+ .description("Some zip archives contain stored entries with data
descriptors which by spec should not " +
+ "happen. If this property is true they will be read
anyway. If false and such an entry is discovered " +
+ "the zip will fail to process.")
+ .required(true)
+ .defaultValue("false")
+ .sensitive(false)
+ .allowableValues("true", "false")
+ .dependsOn(PACKAGING_FORMAT, PackageFormat.ZIP_FORMAT.toString())
+ .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
+ .build();
+
public static final Relationship REL_SUCCESS = new Relationship.Builder()
.name("success")
.description("Unpacked FlowFiles are sent to this relationship")
@@ -195,6 +209,7 @@ public class UnpackContent extends AbstractProcessor {
properties.add(PACKAGING_FORMAT);
properties.add(FILE_FILTER);
properties.add(PASSWORD);
+ properties.add(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
this.properties = Collections.unmodifiableList(properties);
}
@@ -224,7 +239,9 @@ public class UnpackContent extends AbstractProcessor {
if (passwordProperty.isSet()) {
password = passwordProperty.getValue().toCharArray();
}
- zipUnpacker = new ZipUnpacker(fileFilter, password);
+ final PropertyValue allowStoredEntriesWithDataDescriptorVal =
context.getProperty(ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR);
+ final boolean allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptorVal.isSet() ?
allowStoredEntriesWithDataDescriptorVal.asBoolean() : false;
+ zipUnpacker = new ZipUnpacker(fileFilter, password,
allowStoredEntriesWithDataDescriptor);
}
}
@@ -390,17 +407,19 @@ public class UnpackContent extends AbstractProcessor {
private static class ZipUnpacker extends Unpacker {
private final char[] password;
+ private final boolean allowStoredEntriesWithDataDescriptor;
- public ZipUnpacker(final Pattern fileFilter, final char[] password) {
+ public ZipUnpacker(final Pattern fileFilter, final char[] password,
final boolean allowStoredEntriesWithDataDescriptor) {
super(fileFilter);
this.password = password;
+ this.allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptor;
}
@Override
public void unpack(final ProcessSession session, final FlowFile
source, final List<FlowFile> unpacked) {
final String fragmentId = UUID.randomUUID().toString();
if (password == null) {
- session.read(source, new
CompressedZipInputStreamCallback(fileFilter, session, source, unpacked,
fragmentId));
+ session.read(source, new
CompressedZipInputStreamCallback(fileFilter, session, source, unpacked,
fragmentId, allowStoredEntriesWithDataDescriptor));
} else {
session.read(source, new
EncryptedZipInputStreamCallback(fileFilter, session, source, unpacked,
fragmentId, password));
}
@@ -466,19 +485,24 @@ public class UnpackContent extends AbstractProcessor {
}
private static class CompressedZipInputStreamCallback extends
ZipInputStreamCallback {
+
+ private boolean allowStoredEntriesWithDataDescriptor;
+
private CompressedZipInputStreamCallback(
final Pattern fileFilter,
final ProcessSession session,
final FlowFile sourceFlowFile,
final List<FlowFile> unpacked,
- final String fragmentId
+ final String fragmentId,
+ final boolean allowStoredEntriesWithDataDescriptor
) {
super(fileFilter, session, sourceFlowFile, unpacked,
fragmentId);
+ this.allowStoredEntriesWithDataDescriptor =
allowStoredEntriesWithDataDescriptor;
}
@Override
public void process(final InputStream inputStream) throws
IOException {
- try (final ZipArchiveInputStream zipInputStream = new
ZipArchiveInputStream(new BufferedInputStream(inputStream))) {
+ try (final ZipArchiveInputStream zipInputStream = new
ZipArchiveInputStream(new BufferedInputStream(inputStream), null, true,
allowStoredEntriesWithDataDescriptor)) {
ZipArchiveEntry zipEntry;
while ((zipEntry = zipInputStream.getNextZipEntry()) !=
null) {
processEntry(zipInputStream, zipEntry.isDirectory(),
zipEntry.getName(), EncryptionMethod.NONE);
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
index 14053e5f8c..392ebda777 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestUnpackContent.java
@@ -155,6 +155,7 @@ public class TestUnpackContent {
final TestRunner unpackRunner = TestRunners.newTestRunner(new
UnpackContent());
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new
UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"true"); //just forces this to be exercised
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
unpackRunner.enqueue(dataPath.resolve("data.zip"));
unpackRunner.enqueue(dataPath.resolve("data.zip"));
@@ -192,6 +193,7 @@ public class TestUnpackContent {
final TestRunner unpackRunner = TestRunners.newTestRunner(new
UnpackContent());
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new
UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"false");
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
unpackRunner.enqueue(dataPath.resolve("invalid_data.zip"));
unpackRunner.enqueue(dataPath.resolve("invalid_data.zip"));
@@ -253,6 +255,8 @@ public class TestUnpackContent {
final TestRunner autoUnpackRunner = TestRunners.newTestRunner(new
UnpackContent());
unpackRunner.setProperty(UnpackContent.FILE_FILTER,
"^folder/date.txt$");
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"false");
+
autoUnpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.AUTO_DETECT_FORMAT.toString());
autoUnpackRunner.setProperty(UnpackContent.FILE_FILTER,
"^folder/cal.txt$");
unpackRunner.enqueue(dataPath.resolve("data.zip"));
@@ -387,6 +391,7 @@ public class TestUnpackContent {
public void testZipThenMerge() throws IOException {
final TestRunner unpackRunner = TestRunners.newTestRunner(new
UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"false");
unpackRunner.enqueue(dataPath.resolve("data.zip"));
unpackRunner.run();
@@ -424,6 +429,7 @@ public class TestUnpackContent {
public void testZipHandlesBadData() throws IOException {
final TestRunner unpackRunner = TestRunners.newTestRunner(new
UnpackContent());
unpackRunner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
+
unpackRunner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"false");
unpackRunner.enqueue(dataPath.resolve("data.tar"));
unpackRunner.run();
@@ -480,7 +486,7 @@ public class TestUnpackContent {
private void runZipEncryptionMethod(final EncryptionMethod
encryptionMethod) throws IOException {
final TestRunner runner = TestRunners.newTestRunner(new
UnpackContent());
runner.setProperty(UnpackContent.PACKAGING_FORMAT,
UnpackContent.PackageFormat.ZIP_FORMAT.toString());
-
+
runner.setProperty(UnpackContent.ALLOW_STORED_ENTRIES_WITH_DATA_DESCRIPTOR,
"false");
final String password = String.class.getSimpleName();
runner.setProperty(UnpackContent.PASSWORD, password);
