Author: frm
Date: Mon Feb 13 14:26:12 2017
New Revision: 1782777
URL: http://svn.apache.org/viewvc?rev=1782777&view=rev
Log:
OAK-5632 - Expose segment write statistics from IOMonitor
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitor.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitorAdapter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Mon Feb 13 14:26:12 2017
@@ -217,7 +217,7 @@ public class FileStore extends AbstractF
if (indices.length > 0) {
writeNumber = indices[indices.length - 1] + 1;
}
- this.tarWriter = new TarWriter(directory, stats, writeNumber);
+ this.tarWriter = new TarWriter(directory, stats, writeNumber,
ioMonitor);
this.snfeListener = builder.getSnfeListener();
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitor.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitor.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitor.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitor.java
Mon Feb 13 14:26:12 2017
@@ -25,4 +25,8 @@ public interface IOMonitor {
void afterSegmentRead(File file, long msb, long lsb, int length, long
elapsed);
+ void beforeSegmentWrite(File file, long msb, long lsb, int length);
+
+ void afterSegmentWrite(File file, long msb, long lsb, int length, long
elapsed);
+
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitorAdapter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitorAdapter.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitorAdapter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/IOMonitorAdapter.java
Mon Feb 13 14:26:12 2017
@@ -31,4 +31,14 @@ public class IOMonitorAdapter implements
// Intentionally left blank
}
+ @Override
+ public void beforeSegmentWrite(File file, long msb, long lsb, int length) {
+ // Intentionally left blank
+ }
+
+ @Override
+ public void afterSegmentWrite(File file, long msb, long lsb, int length,
long elapsed) {
+ // Intentionally left blank
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Mon Feb 13 14:26:12 2017
@@ -133,7 +133,7 @@ class TarReader implements Closeable {
// regenerate the first generation based on the recovered data
File file = sorted.values().iterator().next();
- generateTarFile(entries, file, recovery);
+ generateTarFile(entries, file, recovery, ioMonitor);
reader = openFirstFileWithValidIndex(singletonList(file),
memoryMapping, ioMonitor);
if (reader != null) {
@@ -161,7 +161,7 @@ class TarReader implements Closeable {
LinkedHashMap<UUID, byte[]> entries = newLinkedHashMap();
collectFileEntries(file, entries, false);
file = findAvailGen(file, ".ro.bak");
- generateTarFile(entries, file, recovery);
+ generateTarFile(entries, file, recovery, ioMonitor);
reader = openFirstFileWithValidIndex(singletonList(file),
memoryMapping, ioMonitor);
if (reader != null) {
return reader;
@@ -207,10 +207,10 @@ class TarReader implements Closeable {
* @param file
* @throws IOException
*/
- private static void generateTarFile(LinkedHashMap<UUID, byte[]> entries,
File file, TarRecovery recovery) throws IOException {
+ private static void generateTarFile(LinkedHashMap<UUID, byte[]> entries,
File file, TarRecovery recovery, IOMonitor ioMonitor) throws IOException {
log.info("Regenerating tar file {}", file);
- try (TarWriter writer = new TarWriter(file)) {
+ try (TarWriter writer = new TarWriter(file, ioMonitor)) {
for (Entry<UUID, byte[]> entry : entries.entrySet()) {
try {
recovery.recoverEntry(entry.getKey(), entry.getValue(),
writer);
@@ -843,7 +843,7 @@ class TarReader implements Closeable {
name.substring(0, pos) + (char) (generation + 1) + ".tar");
log.debug("Writing new generation {}", newFile.getName());
- TarWriter writer = new TarWriter(newFile);
+ TarWriter writer = new TarWriter(newFile, ioMonitor);
for (TarEntry entry : entries) {
if (entry != null) {
long msb = entry.msb();
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarWriter.java
Mon Feb 13 14:26:12 2017
@@ -42,9 +42,11 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;
import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -161,20 +163,23 @@ class TarWriter implements Closeable {
*/
private final Map<UUID, Set<UUID>> graph = newHashMap();
+ private final IOMonitor ioMonitor;
+
/**
* Used for maintenance operations (GC or recovery) via the TarReader and
tests
*/
- TarWriter(File file) {
+ TarWriter(File file, IOMonitor ioMonitor) {
this.file = file;
this.monitor = FileStoreMonitor.DEFAULT;
this.writeIndex = -1;
+ this.ioMonitor = ioMonitor;
}
- TarWriter(File directory, FileStoreMonitor monitor, int writeIndex) {
- this.file = new File(directory, format(FILE_NAME_FORMAT, writeIndex,
- "a"));
+ TarWriter(File directory, FileStoreMonitor monitor, int writeIndex,
IOMonitor ioMonitor) {
+ this.file = new File(directory, format(FILE_NAME_FORMAT, writeIndex,
"a"));
this.monitor = monitor;
this.writeIndex = writeIndex;
+ this.ioMonitor = ioMonitor;
}
/**
@@ -237,31 +242,39 @@ class TarWriter implements Closeable {
return writeEntry(uuid, header, data, offset, size, generation);
}
- private synchronized long writeEntry(
- UUID uuid, byte[] header, byte[] data, int offset, int size, int
generation)
- throws IOException {
+ private synchronized long writeEntry(UUID uuid, byte[] header, byte[]
data, int offset, int size, int generation) throws IOException {
checkState(!closed);
+
if (access == null) {
access = new RandomAccessFile(file, "rw");
channel = access.getChannel();
}
+ long msb = uuid.getMostSignificantBits();
+ long lsb = uuid.getLeastSignificantBits();
+
+ int padding = getPaddingSize(size);
+
long initialLength = access.getFilePointer();
+
access.write(header);
+
+ ioMonitor.beforeSegmentWrite(file, msb, lsb, size);
+ Stopwatch stopwatch = Stopwatch.createStarted();
access.write(data, offset, size);
- int padding = getPaddingSize(size);
+ ioMonitor.afterSegmentWrite(file, msb, lsb, size,
stopwatch.elapsed(TimeUnit.MILLISECONDS));
+
if (padding > 0) {
access.write(ZERO_BYTES, 0, padding);
}
long currentLength = access.getFilePointer();
+ monitor.written(currentLength - initialLength);
+
checkState(currentLength <= Integer.MAX_VALUE);
- TarEntry entry = new TarEntry(
- uuid.getMostSignificantBits(), uuid.getLeastSignificantBits(),
- (int) (currentLength - size - padding), size, generation);
+ TarEntry entry = new TarEntry(msb, lsb, (int) (currentLength - size -
padding), size, generation);
index.put(uuid, entry);
- monitor.written(currentLength - initialLength);
return currentLength;
}
@@ -375,7 +388,7 @@ class TarWriter implements Closeable {
}
close();
int newIndex = writeIndex + 1;
- return new TarWriter(file.getParentFile(), monitor, newIndex);
+ return new TarWriter(file.getParentFile(), monitor, newIndex,
ioMonitor);
}
private void writeBinaryReferences() throws IOException {
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java
Mon Feb 13 14:26:12 2017
@@ -85,15 +85,10 @@ public class FileStoreStatsTest {
long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672
byte[] data = "Hello, World!".getBytes(UTF_8);
- TarWriter writer = null;
- try {
- writer = new TarWriter(directory, stats, 0);
+ try (TarWriter writer = new TarWriter(directory, stats, 0, new
IOMonitorAdapter())) {
writer.writeEntry(msb, lsb, data, 0, data.length, 0);
- } finally {
- writer.close();
+ assertEquals(stats.getApproximateSize() - initialSize,
writer.fileLength());
}
- assertEquals(stats.getApproximateSize() - initialSize,
- writer.fileLength());
} finally {
store.close();
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarFileTest.java
Mon Feb 13 14:26:12 2017
@@ -58,34 +58,21 @@ public class TarFileTest {
long lsb = id.getLeastSignificantBits() & (-1 >>> 4); // OAK-1672
byte[] data = "Hello, World!".getBytes(UTF_8);
- TarWriter writer = new TarWriter(file);
- try {
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
writer.writeEntry(msb, lsb, data, 0, data.length, 0);
assertEquals(ByteBuffer.wrap(data), writer.readEntry(msb, lsb));
- } finally {
- writer.close();
}
assertEquals(5120, file.length());
- TarReader reader = TarReader.open(file, false, new IOMonitorAdapter());
- try {
- assertEquals(ByteBuffer.wrap(data), reader.readEntry(msb, lsb));
- } finally {
- reader.close();
- }
-
- reader = TarReader.open(file, false, new IOMonitorAdapter());
- try {
+ try (TarReader reader = TarReader.open(file, false, new
IOMonitorAdapter())) {
assertEquals(ByteBuffer.wrap(data), reader.readEntry(msb, lsb));
- } finally {
- reader.close();
}
}
@Test
public void testWriteAndReadBinaryReferences() throws Exception {
- try (TarWriter writer = new TarWriter(file)) {
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
writer.writeEntry(0x00, 0x00, new byte[] {0x01, 0x02, 0x3}, 0, 3,
0);
writer.addBinaryReference(1, new UUID(1, 0), "r0");
@@ -132,7 +119,7 @@ public class TarFileTest {
@Test
public void binaryReferencesIndexShouldBeTrimmedDownOnSweep() throws
Exception {
- try (TarWriter writer = new TarWriter(file)) {
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
writer.writeEntry(1, 1, new byte[] {1}, 0, 1, 1);
writer.writeEntry(1, 2, new byte[] {1}, 0, 1, 1);
writer.writeEntry(2, 1, new byte[] {1}, 0, 1, 2);
@@ -168,7 +155,7 @@ public class TarFileTest {
@Test
public void graphShouldBeTrimmedDownOnSweep() throws Exception {
- try (TarWriter writer = new TarWriter(file)) {
+ try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
writer.writeEntry(1, 1, new byte[] {1}, 0, 1, 1);
writer.writeEntry(1, 2, new byte[] {1}, 0, 1, 1);
writer.writeEntry(1, 3, new byte[] {1}, 0, 1, 1);
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java?rev=1782777&r1=1782776&r2=1782777&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarWriterTest.java
Mon Feb 13 14:26:12 2017
@@ -36,7 +36,6 @@ import java.util.Set;
import java.util.UUID;
import com.google.common.collect.ImmutableList;
-
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
@@ -78,7 +77,7 @@ public class TarWriterTest {
assertTrue(z.getReferences().isEmpty());
File tar = folder.newFile(getClass().getName() + ".tar");
- try (TarWriter tarWriter = new TarWriter(tar)) {
+ try (TarWriter tarWriter = new TarWriter(tar, new IOMonitorAdapter()))
{
y.write(tarWriter);
b.write(tarWriter);
a.write(tarWriter);
@@ -239,8 +238,7 @@ public class TarWriterTest {
@Test
public void createNextGenerationTest() throws IOException {
int counter = 2222;
- TarWriter t0 = new TarWriter(folder.newFolder(),
- FileStoreMonitor.DEFAULT, counter);
+ TarWriter t0 = new TarWriter(folder.newFolder(),
FileStoreMonitor.DEFAULT, counter, new IOMonitorAdapter());
// not dirty, will not create a new writer
TarWriter t1 = t0.createNextGeneration();
