GWphua commented on code in PR #18982:
URL: https://github.com/apache/druid/pull/18982#discussion_r2802071695
##########
processing/src/main/java/org/apache/druid/utils/CompressionUtils.java:
##########
@@ -212,6 +286,106 @@ public static long zip(File directory, OutputStream out)
throws IOException
return totalSize;
}
+ /**
+ * Compresses directory contents using LZ4 block compression with a simple
archive format.
+ * Format: [file_count:4
bytes][file1_name_length:4][file1_name:bytes][file1_size:8][file1_data:bytes]...
+ *
+ * @param directory The directory whose contents should be compressed
+ * @param out The output stream to write compressed data to
+ * @return The number of bytes (uncompressed) read from the input directory
+ */
+ public static long lz4CompressDirectory(File directory, OutputStream out)
throws IOException
+ {
+ if (!directory.isDirectory()) {
+ throw new IOE("directory[%s] is not a directory", directory);
+ }
+
+ // Use fast compressor for better performance (lower CPU, faster
compression)
+ final LZ4BlockOutputStream lz4Out = new LZ4BlockOutputStream(
+ out,
+ 64 * 1024, // Block size
+ LZ4Factory.fastestInstance().fastCompressor()
+ );
+ // Use DataOutputStream for structured writing
+ final DataOutputStream dataOut = new DataOutputStream(lz4Out);
+ final File[] files = directory.listFiles();
+
+ if (files == null) {
+ throw new IOE("Cannot list files in directory[%s]", directory);
+ }
+
+ // Sort for consistency
+ final File[] sortedFiles =
Arrays.stream(files).sorted().toArray(File[]::new);
+
+ dataOut.writeInt(sortedFiles.length);
+
+ long totalSize = 0;
+
+ for (File file : sortedFiles) {
+ if (file.isDirectory()) {
+ continue; // Skip subdirectories like ZIP does
+ }
+
+ log.debug("Compressing file[%s] with size[%,d]. Total size so far[%,d]",
file, file.length(), totalSize);
+
+ final String fileName = file.getName();
+ final byte[] fileNameBytes = fileName.getBytes(StandardCharsets.UTF_8);
+ dataOut.writeInt(fileNameBytes.length);
+ dataOut.write(fileNameBytes);
+
+ final long fileSize = file.length();
+ if (fileSize > Integer.MAX_VALUE) {
+ throw new IOE("file[%s] too large [%,d]", file, fileSize);
+ }
+
+ dataOut.writeLong(fileSize);
+ totalSize += fileSize;
+
+ // Copy file content to dataOut
+ try (FileInputStream fileIn = new FileInputStream(file)) {
+ ByteStreams.copy(fileIn, dataOut);
+ }
+ }
+
+ dataOut.flush();
+ lz4Out.finish();
+ return totalSize;
+ }
+
+ /**
+ * Decompresses LZ4-compressed directory archive
+ */
+ public static FileUtils.FileCopyResult lz4DecompressDirectory(InputStream
in, File outDir) throws IOException
+ {
+ if (!(outDir.exists() && outDir.isDirectory())) {
+ throw new ISE("outDir[%s] must exist and be a directory", outDir);
+ }
+
+ final LZ4BlockInputStream lz4In = new LZ4BlockInputStream(in);
Review Comment:
Seems like the deprecated API comes from lz4-java 1.10.1, which is not
introduced into the repo yet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
