This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new f9e7bc4cffc HDDS-15836. Add Archiver.appendFile for incremental TAR
writes (#10735)
f9e7bc4cffc is described below
commit f9e7bc4cffc10e1d4501064da12ecaf1a957a563
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Tue Jul 14 18:25:58 2026 +0530
HDDS-15836. Add Archiver.appendFile for incremental TAR writes (#10735)
Co-authored-by: Cursor <[email protected]>
---
.../org/apache/hadoop/hdds/utils/Archiver.java | 76 ++++++++++++++++++++++
.../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 ++++++++++++
2 files changed, 116 insertions(+)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java
index 4f95df12776..d306295b2b4 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java
@@ -25,10 +25,13 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
import java.util.stream.Stream;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
@@ -37,6 +40,7 @@
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.tar.TarConstants;
+import org.apache.commons.compress.archivers.tar.TarUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -48,6 +52,9 @@ public final class Archiver {
static final int MIN_BUFFER_SIZE = 8 * (int) OzoneConsts.KB; // same as
IOUtils.DEFAULT_BUFFER_SIZE
static final int MAX_BUFFER_SIZE = (int) OzoneConsts.MB;
+ private static final byte[] TAR_ZERO_BLOCK = new
byte[TarConstants.DEFAULT_RCDSIZE];
+ private static final int TAR_SIZE_OFFSET = TarConstants.NAMELEN +
TarConstants.MODELEN
+ + TarConstants.UIDLEN + TarConstants.GIDLEN;
private static final Logger LOG = LoggerFactory.getLogger(Archiver.class);
private Archiver() {
@@ -61,6 +68,49 @@ public static void create(File tarFile, Path from) throws
IOException {
}
}
+ /**
+ * Opens a tarball for incremental appends. The stream stays open until
+ * {@link AppendableTar#close()} writes the end-of-archive marker.
+ */
+ public static AppendableTar openForAppend(File tarFile) throws IOException {
+ return new AppendableTar(tarFile);
+ }
+
+ /**
+ * Remove the tar end-of-archive marker so new entries can be appended.
+ */
+ private static void stripTarEofMarker(File tarFile) throws IOException {
+ try (RandomAccessFile raf = new RandomAccessFile(tarFile, "rw")) {
+ long position = 0;
+ long fileLength = raf.length();
+ byte[] header = new byte[TarConstants.DEFAULT_RCDSIZE];
+ while (position + TarConstants.DEFAULT_RCDSIZE <= fileLength) {
+ raf.seek(position);
+ raf.readFully(header);
+ if (Arrays.equals(header, TAR_ZERO_BLOCK)) {
+ raf.setLength(position);
+ return;
+ }
+ long entrySize = parseTarEntrySize(header);
+ position += TarConstants.DEFAULT_RCDSIZE +
paddedTarEntrySize(entrySize);
+ }
+ throw new IOException("Invalid tar archive without an end-of-archive
marker: " + tarFile);
+ }
+ }
+
+ private static long parseTarEntrySize(byte[] header) throws IOException {
+ try {
+ return TarUtils.parseOctalOrBinary(header, TAR_SIZE_OFFSET,
TarConstants.SIZELEN);
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Invalid tar entry size.", e);
+ }
+ }
+
+ private static long paddedTarEntrySize(long size) {
+ long recordSize = TarConstants.DEFAULT_RCDSIZE;
+ return ((size + recordSize - 1) / recordSize) * recordSize;
+ }
+
/** Extract {@code tarFile} to {@code dir}. */
public static void extract(File tarFile, Path dir) throws IOException {
Files.createDirectories(dir);
@@ -220,4 +270,30 @@ static int getBufferSize(long fileSize) {
return Math.toIntExact(Math.min(MAX_BUFFER_SIZE, Math.max(fileSize,
MIN_BUFFER_SIZE)));
}
+ /** Incrementally append entries to a tarball. */
+ public static final class AppendableTar implements AutoCloseable {
+ private final ArchiveOutputStream<TarArchiveEntry> out;
+
+ private AppendableTar(File tarFile) throws IOException {
+ OutputStream fos;
+ if (tarFile.exists() && tarFile.length() > 0) {
+ stripTarEofMarker(tarFile);
+ fos = Files.newOutputStream(tarFile.toPath(),
StandardOpenOption.WRITE, StandardOpenOption.APPEND);
+ } else {
+ fos = Files.newOutputStream(tarFile.toPath(),
StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ }
+ out = tar(fos);
+ }
+
+ public void appendFile(File file, String entryName) throws IOException {
+ includeFile(file, entryName, out);
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.finish();
+ out.close();
+ }
+ }
+
}
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java
index 6a361d79664..b58d1504ec8 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java
@@ -133,4 +133,44 @@ void testLinkAndIncludeFileFailedHardLink() throws
IOException {
Files.deleteIfExists(tmpDir);
}
+ @Test
+ void appendFileCreatesAndExtendsTar() throws IOException {
+ Path tmpDir = Files.createTempDirectory("archiver-append");
+ File tarFile = tmpDir.resolve("export.tar").toFile();
+ File part1 = tmpDir.resolve("part001.txt").toFile();
+ File part2 = tmpDir.resolve("part002.txt").toFile();
+ Files.write(part1.toPath(), "1\n2\n".getBytes(StandardCharsets.UTF_8));
+ Files.write(part2.toPath(), "3\n".getBytes(StandardCharsets.UTF_8));
+
+ try (Archiver.AppendableTar tar = Archiver.openForAppend(tarFile)) {
+ tar.appendFile(part1, "part001.txt");
+ tar.appendFile(part2, "part002.txt");
+ }
+
+ Path extractDir = tmpDir.resolve("extract");
+ Archiver.extract(tarFile, extractDir);
+
assertThat(extractDir.resolve("part001.txt")).hasSameBinaryContentAs(part1.toPath());
+
assertThat(extractDir.resolve("part002.txt")).hasSameBinaryContentAs(part2.toPath());
+ }
+
+ @Test
+ void appendFilePreservesZeroBlocksAtEndOfEntry() throws IOException {
+ Path tmpDir = Files.createTempDirectory("archiver-append-zero-block");
+ File tarFile = tmpDir.resolve("export.tar").toFile();
+ File part1 = tmpDir.resolve("part001.bin").toFile();
+ File part2 = tmpDir.resolve("part002.txt").toFile();
+ byte[] zeroBlockPayload = new byte[1024];
+ Files.write(part1.toPath(), zeroBlockPayload);
+ Files.write(part2.toPath(), "next\n".getBytes(StandardCharsets.UTF_8));
+
+ try (Archiver.AppendableTar tar = Archiver.openForAppend(tarFile)) {
+ tar.appendFile(part1, "part001.bin");
+ tar.appendFile(part2, "part002.txt");
+ }
+
+ Path extractDir = tmpDir.resolve("extract");
+ Archiver.extract(tarFile, extractDir);
+
assertThat(extractDir.resolve("part001.bin")).hasSameBinaryContentAs(part1.toPath());
+
assertThat(extractDir.resolve("part002.txt")).hasSameBinaryContentAs(part2.toPath());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
