This is an automated email from the ASF dual-hosted git repository.
mthomsen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 3e60414d48 NIFI-10436 Initial basic brotli support NIFI-10436 Add
Brotli-compressed SampleFile.txt.br for unit testing NIFI-10436 Fixup basic
usage of Brotli4j API NIFI-10436 NOTICE ok, LICENSE updated NIFI-10436 style
fixup CompressContent.java NIFI-10436 Update CompressContent.java
3e60414d48 is described below
commit 3e60414d486342ee59cee65818c79f3612946d3d
Author: Matthew Hawkins <[email protected]>
AuthorDate: Sat Sep 17 21:13:30 2022 +1000
NIFI-10436 Initial basic brotli support
NIFI-10436 Add Brotli-compressed SampleFile.txt.br for unit testing
NIFI-10436 Fixup basic usage of Brotli4j API
NIFI-10436 NOTICE ok, LICENSE updated
NIFI-10436 style fixup CompressContent.java
NIFI-10436 Update CompressContent.java
This closes #6432
Signed-off-by: Mike Thomsen <[email protected]>
---
nifi-assembly/LICENSE | 5 +++-
.../nifi-standard-processors/pom.xml | 5 ++++
.../nifi/processors/standard/CompressContent.java | 29 +++++++++++++++++---
.../processors/standard/TestCompressContent.java | 30 +++++++++++++++++++++
.../resources/CompressedData/SampleFile.txt.br | Bin 0 -> 71 bytes
nifi-nar-bundles/nifi-standard-bundle/pom.xml | 5 ++++
6 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/nifi-assembly/LICENSE b/nifi-assembly/LICENSE
index 4ecd35df8a..e6842290a4 100644
--- a/nifi-assembly/LICENSE
+++ b/nifi-assembly/LICENSE
@@ -3120,4 +3120,7 @@ which is available under an MIT license.
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
\ No newline at end of file
+ THE SOFTWARE.
+
+This product bundles 'Brotli4j' which is available under the Apache License 2.0
+
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
index 3ea60c1647..0a7b197ae8 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/pom.xml
@@ -229,6 +229,11 @@
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.aayushatharva.brotli4j</groupId>
+ <artifactId>brotli4j</artifactId>
+ <version>1.8.0</version>
+ </dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
index 6ad4192cd1..605b25cf97 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/CompressContent.java
@@ -25,6 +25,10 @@ import
org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import
org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorInputStream;
import
org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
import
org.apache.commons.compress.compressors.zstandard.ZstdCompressorOutputStream;
+import com.aayushatharva.brotli4j.decoder.BrotliInputStream;
+import com.aayushatharva.brotli4j.encoder.BrotliOutputStream;
+import com.aayushatharva.brotli4j.Brotli4jLoader;
+import com.aayushatharva.brotli4j.encoder.Encoder;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
@@ -84,7 +88,7 @@ import java.util.zip.InflaterInputStream;
@SideEffectFree
@SupportsBatching
@InputRequirement(Requirement.INPUT_REQUIRED)
-@Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma",
"xz-lzma2", "snappy", "snappy-hadoop", "snappy framed", "lz4-framed",
"deflate", "zstd"})
+@Tags({"content", "compress", "decompress", "gzip", "bzip2", "lzma",
"xz-lzma2", "snappy", "snappy-hadoop", "snappy framed", "lz4-framed",
"deflate", "zstd", "brotli"})
@CapabilityDescription("Compresses or decompresses the contents of FlowFiles
using a user-specified compression algorithm and updates the mime.type "
+ "attribute as appropriate. This processor operates in a very memory
efficient way so very large objects well beyond the heap size "
+ "are generally fine to process")
@@ -107,16 +111,17 @@ public class CompressContent extends AbstractProcessor {
public static final String COMPRESSION_FORMAT_SNAPPY_FRAMED = "snappy
framed";
public static final String COMPRESSION_FORMAT_LZ4_FRAMED ="lz4-framed";
public static final String COMPRESSION_FORMAT_ZSTD = "zstd";
+ public static final String COMPRESSION_FORMAT_BROTLI = "brotli";
public static final String MODE_COMPRESS = "compress";
public static final String MODE_DECOMPRESS = "decompress";
public static final PropertyDescriptor COMPRESSION_FORMAT = new
PropertyDescriptor.Builder()
.name("Compression Format")
- .description("The compression format to use. Valid values are: GZIP,
Deflate, ZSTD, BZIP2, XZ-LZMA2, LZMA, Snappy, Snappy Hadoop, Snappy Framed, and
LZ4-Framed")
+ .description("The compression format to use. Valid values are: GZIP,
Deflate, ZSTD, BZIP2, XZ-LZMA2, LZMA, Brotli, Snappy, Snappy Hadoop, Snappy
Framed, and LZ4-Framed")
.allowableValues(COMPRESSION_FORMAT_ATTRIBUTE,
COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE, COMPRESSION_FORMAT_BZIP2,
COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_LZMA,
COMPRESSION_FORMAT_SNAPPY, COMPRESSION_FORMAT_SNAPPY_HADOOP,
COMPRESSION_FORMAT_SNAPPY_FRAMED,
- COMPRESSION_FORMAT_LZ4_FRAMED, COMPRESSION_FORMAT_ZSTD)
+ COMPRESSION_FORMAT_LZ4_FRAMED, COMPRESSION_FORMAT_ZSTD,
COMPRESSION_FORMAT_BROTLI)
.defaultValue(COMPRESSION_FORMAT_ATTRIBUTE)
.required(true)
.build();
@@ -135,7 +140,8 @@ public class CompressContent extends AbstractProcessor {
.defaultValue("1")
.required(true)
.allowableValues("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
- .dependsOn(COMPRESSION_FORMAT, COMPRESSION_FORMAT_ATTRIBUTE,
COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE,
COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_ZSTD)
+ .dependsOn(COMPRESSION_FORMAT, COMPRESSION_FORMAT_ATTRIBUTE,
COMPRESSION_FORMAT_GZIP, COMPRESSION_FORMAT_DEFLATE,
+ COMPRESSION_FORMAT_XZ_LZMA2, COMPRESSION_FORMAT_ZSTD,
COMPRESSION_FORMAT_BROTLI)
.dependsOn(MODE, MODE_COMPRESS)
.build();
@@ -188,6 +194,7 @@ public class CompressContent extends AbstractProcessor {
mimeTypeMap.put("application/x-snappy-framed",
COMPRESSION_FORMAT_SNAPPY_FRAMED);
mimeTypeMap.put("application/x-lz4-framed",
COMPRESSION_FORMAT_LZ4_FRAMED);
mimeTypeMap.put("application/zstd", COMPRESSION_FORMAT_ZSTD);
+ mimeTypeMap.put("application/x-brotli", COMPRESSION_FORMAT_BROTLI);
this.compressionFormatMimeTypeMap =
Collections.unmodifiableMap(mimeTypeMap);
}
@@ -280,6 +287,9 @@ public class CompressContent extends AbstractProcessor {
case COMPRESSION_FORMAT_ZSTD:
fileExtension = ".zst";
break;
+ case COMPRESSION_FORMAT_BROTLI:
+ fileExtension = ".br";
+ break;
default:
fileExtension = "";
break;
@@ -340,6 +350,13 @@ public class CompressContent extends AbstractProcessor {
compressionOut = new
ZstdCompressorOutputStream(bufferedOut, zstdcompressionLevel);
mimeTypeRef.set("application/zstd");
break;
+ case COMPRESSION_FORMAT_BROTLI:
+ Brotli4jLoader.ensureAvailability();
+ compressionLevel =
context.getProperty(COMPRESSION_LEVEL).asInteger();
+ Encoder.Parameters params = new
Encoder.Parameters().setQuality(compressionLevel);
+ compressionOut = new
BrotliOutputStream(bufferedOut, params);
+ mimeTypeRef.set("application/x-brotli");
+ break;
case COMPRESSION_FORMAT_BZIP2:
default:
mimeTypeRef.set("application/x-bzip2");
@@ -379,6 +396,10 @@ public class CompressContent extends AbstractProcessor {
case COMPRESSION_FORMAT_ZSTD:
compressionIn = new
ZstdCompressorInputStream(bufferedIn);
break;
+ case COMPRESSION_FORMAT_BROTLI:
+ Brotli4jLoader.ensureAvailability();
+ compressionIn = new
BrotliInputStream(bufferedIn);
+ break;
default:
compressionIn = new
CompressorStreamFactory().createCompressorInputStream(compressionFormat.toLowerCase(),
bufferedIn);
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java
index 0d8727a8e3..a1cb5feadc 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestCompressContent.java
@@ -377,4 +377,34 @@ public class TestCompressContent {
flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt"));
flowFile.assertAttributeEquals("filename", "SampleFile.txt");
}
+
+ @Test
+ public void testBrotliCompress() throws Exception {
+ final TestRunner runner =
TestRunners.newTestRunner(CompressContent.class);
+ runner.setProperty(CompressContent.MODE,
CompressContent.MODE_COMPRESS);
+ runner.setProperty(CompressContent.COMPRESSION_FORMAT,
CompressContent.COMPRESSION_FORMAT_BROTLI);
+ runner.setProperty(CompressContent.UPDATE_FILENAME, "true");
+
+
runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile.txt"));
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1);
+ MockFlowFile flowFile =
runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0);
+ flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(),
"application/x-brotli");
+ flowFile.assertAttributeEquals("filename", "SampleFile.txt.br");
+ }
+
+ @Test
+ public void testBrotliDecompress() throws Exception {
+ final TestRunner runner =
TestRunners.newTestRunner(CompressContent.class);
+ runner.setProperty(CompressContent.MODE,
CompressContent.MODE_DECOMPRESS);
+ runner.setProperty(CompressContent.COMPRESSION_FORMAT,
CompressContent.COMPRESSION_FORMAT_BROTLI);
+ runner.setProperty(CompressContent.UPDATE_FILENAME, "true");
+
runner.enqueue(Paths.get("src/test/resources/CompressedData/SampleFile.txt.br"));
+ runner.run();
+ runner.assertAllFlowFilesTransferred(CompressContent.REL_SUCCESS, 1);
+ MockFlowFile flowFile =
runner.getFlowFilesForRelationship(CompressContent.REL_SUCCESS).get(0);
+
flowFile.assertContentEquals(Paths.get("src/test/resources/CompressedData/SampleFile.txt"));
+ flowFile.assertAttributeEquals("filename", "SampleFile.txt");
+ }
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br
new file mode 100755
index 0000000000..dd2bbc8f8d
Binary files /dev/null and
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/resources/CompressedData/SampleFile.txt.br
differ
diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml
b/nifi-nar-bundles/nifi-standard-bundle/pom.xml
index 1890838f68..166b013ff3 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml
+++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml
@@ -159,6 +159,11 @@
<artifactId>zstd-jni</artifactId>
<version>1.5.2-3</version>
</dependency>
+ <dependency>
+ <groupId>com.aayushatharva.brotli4j</groupId>
+ <artifactId>brotli4j</artifactId>
+ <version>1.8.0</version>
+ </dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
