Author: alexparvulescu Date: Tue Sep 6 13:43:50 2016 New Revision: 1759434
URL: http://svn.apache.org/viewvc?rev=1759434&view=rev Log: OAK-4746 Move writeFile into the TarWriter Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1759434&r1=1759433&r2=1759434&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Sep 6 13:43:50 2016 @@ -129,7 +129,7 @@ public class FileStore implements Segmen private static final Pattern FILE_NAME_PATTERN = Pattern.compile("(data|bulk)((0|[1-9][0-9]*)[0-9]{4})([a-z])?.tar"); - private static final String FILE_NAME_FORMAT = "data%05d%s.tar"; + static final String FILE_NAME_FORMAT = "data%05d%s.tar"; private static final String LOCK_FILE_NAME = "repo.lock"; @@ -165,10 +165,6 @@ public class FileStore implements Segmen private volatile List<TarReader> readers; - private int writeNumber; - - private volatile File writeFile; - private volatile TarWriter tarWriter; private final RandomAccessFile lockFile; @@ -330,14 +326,13 @@ public class FileStore implements Segmen this.stats = new FileStoreStats(builder.getStatsProvider(), this, size()); if (!readOnly) { + int writeNumber = 0; if (indices.length > 0) { - this.writeNumber = indices[indices.length - 1] + 1; - } else { - this.writeNumber = 0; + writeNumber = indices[indices.length - 1] + 1; } - this.writeFile = new File(directory, format( - FILE_NAME_FORMAT, writeNumber, "a")); - this.tarWriter = new TarWriter(writeFile, stats); + this.tarWriter = new TarWriter(directory, stats, writeNumber); + } else { + this.tarWriter = null; } // FIXME OAK-4621: External invocation of background operations @@ -655,7 +650,7 @@ public class FileStore implements Segmen fileStoreLock.readLock().lock(); try { readersSnapshot = ImmutableList.copyOf(readers); - writeFileSnapshotSize = writeFile != null ? writeFile.length() : 0; + writeFileSnapshotSize = tarWriter != null ? tarWriter.fileLength() : 0; } finally { fileStoreLock.readLock().unlock(); } @@ -1452,20 +1447,15 @@ public class FileStore implements Segmen * @throws IOException */ private void newWriter() throws IOException { - if (tarWriter.isDirty()) { - tarWriter.close(); - + TarWriter newWriter = tarWriter.createNextGeneration(); + if (newWriter != tarWriter) { + File writeFile = tarWriter.getFile(); List<TarReader> list = newArrayListWithCapacity(1 + readers.size()); list.add(TarReader.open(writeFile, memoryMapping)); list.addAll(readers); readers = list; - - writeNumber++; - writeFile = new File( - directory, - format(FILE_NAME_FORMAT, writeNumber, "a")); - tarWriter = new TarWriter(writeFile, stats); + tarWriter = newWriter; } } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java?rev=1759434&r1=1759433&r2=1759434&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java Tue Sep 6 13:43:50 2016 @@ -27,6 +27,8 @@ import static com.google.common.collect. import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Maps.newLinkedHashMap; import static com.google.common.collect.Sets.newHashSet; +import static java.lang.String.format; +import static org.apache.jackrabbit.oak.segment.file.FileStore.FILE_NAME_FORMAT; import java.io.Closeable; import java.io.File; @@ -110,6 +112,8 @@ class TarWriter implements Closeable { } } + private final int writeIndex; + /** * The file being written. This instance is also used as an additional * synchronization point by {@link #flush()} and {@link #close()} to @@ -157,13 +161,20 @@ class TarWriter implements Closeable { */ private final Map<UUID, Set<UUID>> graph = newHashMap(); + /** + * Used for maintenance operations (GC or recovery) via the TarReader and tests + */ TarWriter(File file) { - this(file, FileStoreMonitor.DEFAULT); + this.file = file; + this.monitor = FileStoreMonitor.DEFAULT; + this.writeIndex = -1; } - TarWriter(File file, FileStoreMonitor monitor) { - this.file = file; + TarWriter(File directory, FileStoreMonitor monitor, int writeIndex) { + this.file = new File(directory, format(FILE_NAME_FORMAT, writeIndex, + "a")); this.monitor = monitor; + this.writeIndex = writeIndex; } /** @@ -308,10 +319,6 @@ class TarWriter implements Closeable { } } - boolean isDirty() { - return access != null; - } - /** * Closes this tar file. * @@ -352,6 +359,23 @@ class TarWriter implements Closeable { monitor.written(currentPosition - initialPosition); } + /** + * If the current instance is dirty, this will return a new TarWriter based + * on the next generation of the file being written to by incrementing the + * internal {@link #writeIndex} counter. Otherwise it will return the + * current instance. + */ + synchronized TarWriter createNextGeneration() throws IOException { + checkState(writeIndex >= 0); + // If nothing was written to this file, then we're already done. + if (access == null) { + return this; + } + close(); + int newIndex = writeIndex + 1; + return new TarWriter(file.getParentFile(), monitor, newIndex); + } + private void writeBinaryReferences() throws IOException { int binaryReferenceSize = 0; @@ -625,6 +649,18 @@ class TarWriter implements Closeable { } } + synchronized long fileLength() { + return file.length(); + } + + synchronized File getFile() { + return file; + } + + synchronized boolean isClosed() { + return closed; + } + //------------------------------------------------------------< Object >-- @Override Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java?rev=1759434&r1=1759433&r2=1759434&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java Tue Sep 6 13:43:50 2016 @@ -73,31 +73,30 @@ public class FileStoreStatsTest { @Test public void tarWriterIntegration() throws Exception{ StatisticsProvider statsProvider = new DefaultStatisticsProvider(executor); - FileStore store = fileStoreBuilder(segmentFolder.newFolder()) + File directory = segmentFolder.newFolder(); + FileStore store = fileStoreBuilder(directory) .withStatisticsProvider(statsProvider) .build(); - FileStoreStats stats = new FileStoreStats(statsProvider, store, 0); - try { long initialSize = stats.getApproximateSize(); - UUID id = UUID.randomUUID(); long msb = id.getMostSignificantBits(); long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672 byte[] data = "Hello, World!".getBytes(UTF_8); - File file = segmentFolder.newFile(); - try (TarWriter writer = new TarWriter(file, stats)) { + TarWriter writer = null; + try { + writer = new TarWriter(directory, stats, 0); writer.writeEntry(msb, lsb, data, 0, data.length, 0); + } finally { + writer.close(); } - - assertEquals(stats.getApproximateSize() - initialSize, file.length()); - + assertEquals(stats.getApproximateSize() - initialSize, + writer.fileLength()); } finally { store.close(); } - assertEquals(1, stats.getJournalWriteStatsAsCount()); } Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java?rev=1759434&r1=1759433&r2=1759434&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java (original) +++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java Tue Sep 6 13:43:50 2016 @@ -19,11 +19,13 @@ package org.apache.jackrabbit.oak.segment.file; +import static com.google.common.base.Charsets.UTF_8; import static com.google.common.collect.Maps.newHashMap; import static com.google.common.collect.Sets.newHashSet; import static java.nio.ByteBuffer.allocate; import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import java.io.File; @@ -34,6 +36,7 @@ import java.util.Set; import java.util.UUID; import com.google.common.collect.ImmutableList; + import org.apache.jackrabbit.oak.segment.RecordId; import org.apache.jackrabbit.oak.segment.Segment; import org.apache.jackrabbit.oak.segment.SegmentId; @@ -233,4 +236,28 @@ public class TarWriterTest { } } + @Test + public void createNextGenerationTest() throws IOException { + int counter = 2222; + TarWriter t0 = new TarWriter(folder.newFolder(), + FileStoreMonitor.DEFAULT, counter); + + // not dirty, will not create a new writer + TarWriter t1 = t0.createNextGeneration(); + assertEquals(t0, t1); + assertTrue(t1.getFile().getName().contains("" + counter)); + + // dirty, will create a new writer + UUID id = UUID.randomUUID(); + long msb = id.getMostSignificantBits(); + long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672 + byte[] data = "Hello, World!".getBytes(UTF_8); + t1.writeEntry(msb, lsb, data, 0, data.length, 0); + + TarWriter t2 = t1.createNextGeneration(); + assertNotEquals(t1, t2); + assertTrue(t1.isClosed()); + assertTrue(t2.getFile().getName().contains("" + (counter + 1))); + } + }
