github-advanced-security[bot] commented on code in PR #18982:
URL: https://github.com/apache/druid/pull/18982#discussion_r2762108984
##########
processing/src/test/java/org/apache/druid/java/util/common/CompressionUtilsTest.java:
##########
@@ -407,6 +408,46 @@
}
}
+ @Test
+ public void testGoodLz4CompressUncompressDirectory() throws IOException
+ {
+ final File tmpDir =
temporaryFolder.newFolder("testGoodLz4CompressUncompressDirectory");
+ final File lz4File = new File(tmpDir, "compressionUtilTest.lz4");
+
+ try (final OutputStream out = new FileOutputStream(lz4File)) {
+ CompressionUtils.Format.LZ4.compressDirectory(testDir, out);
+ }
+
+ final File newDir = new File(tmpDir, "newDir");
+ Assert.assertTrue(newDir.mkdir());
+
+ final FileUtils.FileCopyResult result;
+ try (final InputStream in = new FileInputStream(lz4File)) {
+ result = CompressionUtils.Format.LZ4.decompressDirectory(in, newDir);
+ }
+
+ verifyUnzip(newDir, result, ImmutableMap.of(testFile.getName(),
StringUtils.toUtf8(CONTENT)));
+ }
+
+ @Test
+ public void testDecompressLz4() throws IOException
+ {
+ final File tmpDir = temporaryFolder.newFolder("testDecompressLz4");
+ final File lz4File = new File(tmpDir, testFile.getName() + ".lz4");
+ Assert.assertFalse(lz4File.exists());
+
+ try (
+ final OutputStream out = new LZ4BlockOutputStream(new
FileOutputStream(lz4File));
+ final InputStream in = new FileInputStream(testFile)
+ ) {
+ ByteStreams.copy(in, out);
+ }
+
+ try (final InputStream inputStream = CompressionUtils.decompress(new
FileInputStream(lz4File), lz4File.getName())) {
Review Comment:
## Potential input resource leak
This FileInputStream is not always closed on method exit.
[Show more
details](https://github.com/apache/druid/security/code-scanning/10799)
##########
processing/src/main/java/org/apache/druid/utils/CompressionUtils.java:
##########
@@ -206,12 +280,112 @@
~ }
~ zipOut.closeEntry();
~ // Workaround for http://hg.openjdk.java.net/jdk8/jdk8/jdk/rev/759aa847dcaf
~ zipOut.flush();
~ zipOut.finish();
~
return totalSize;
}
+ /**
+ * Compresses directory contents using LZ4 block compression with a simple
archive format.
+ * Format: [file_count:4
bytes][file1_name_length:4][file1_name:bytes][file1_size:8][file1_data:bytes]...
+ *
+ * @param directory The directory whose contents should be compressed
+ * @param out The output stream to write compressed data to
+ * @return The number of bytes (uncompressed) read from the input directory
+ */
+ public static long lz4CompressDirectory(File directory, OutputStream out)
throws IOException
+ {
+ if (!directory.isDirectory()) {
+ throw new IOE("directory[%s] is not a directory", directory);
+ }
+
+ // Use fast compressor for better performance (lower CPU, faster
compression)
+ final LZ4BlockOutputStream lz4Out = new LZ4BlockOutputStream(
+ out,
+ 64 * 1024, // Block size
+ LZ4Factory.fastestInstance().fastCompressor()
+ );
+ // Use DataOutputStream for structured writing
+ final DataOutputStream dataOut = new DataOutputStream(lz4Out);
+ final File[] files = directory.listFiles();
+
+ if (files == null) {
+ throw new IOE("Cannot list files in directory[%s]", directory);
+ }
+
+ // Sort for consistency
+ final File[] sortedFiles =
Arrays.stream(files).sorted().toArray(File[]::new);
+
+ dataOut.writeInt(sortedFiles.length);
+
+ long totalSize = 0;
+
+ for (File file : sortedFiles) {
+ if (file.isDirectory()) {
+ continue; // Skip subdirectories like ZIP does
+ }
+
+ log.debug("Compressing file[%s] with size[%,d]. Total size so far[%,d]",
file, file.length(), totalSize);
+
+ final String fileName = file.getName();
+ final byte[] fileNameBytes = fileName.getBytes(StandardCharsets.UTF_8);
+ dataOut.writeInt(fileNameBytes.length);
+ dataOut.write(fileNameBytes);
+
+ final long fileSize = file.length();
+ if (fileSize > Integer.MAX_VALUE) {
+ throw new IOE("file[%s] too large [%,d]", file, fileSize);
+ }
+
+ dataOut.writeLong(fileSize);
+ totalSize += fileSize;
+
+ // Copy file content to dataOut
+ try (FileInputStream fileIn = new FileInputStream(file)) {
+ ByteStreams.copy(fileIn, dataOut);
+ }
+ }
+
+ dataOut.flush();
+ lz4Out.finish();
+ return totalSize;
+ }
+
+ /**
+ * Decompresses LZ4-compressed directory archive
+ */
+ public static FileUtils.FileCopyResult lz4DecompressDirectory(InputStream
in, File outDir) throws IOException
+ {
+ if (!(outDir.exists() && outDir.isDirectory())) {
+ throw new ISE("outDir[%s] must exist and be a directory", outDir);
+ }
+
+ final LZ4BlockInputStream lz4In = new LZ4BlockInputStream(in);
Review Comment:
## Deprecated method or constructor invocation
Invoking [LZ4BlockInputStream.LZ4BlockInputStream](1) should be avoided
because it has been deprecated.
[Show more
details](https://github.com/apache/druid/security/code-scanning/10800)
##########
processing/src/main/java/org/apache/druid/utils/CompressionUtils.java:
##########
@@ -622,6 +811,8 @@
{
if (fileName.endsWith(Format.GZ.getSuffix())) {
return gzipInputStream(in);
+ } else if (fileName.endsWith(Format.LZ4.getSuffix())) {
+ return new LZ4BlockInputStream(in);
Review Comment:
## Deprecated method or constructor invocation
Invoking [LZ4BlockInputStream.LZ4BlockInputStream](1) should be avoided
because it has been deprecated.
[Show more
details](https://github.com/apache/druid/security/code-scanning/10801)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
