Author: jukka
Date: Mon Mar 17 18:50:39 2014
New Revision: 1578510
URL: http://svn.apache.org/r1578510
Log:
OAK-1423: SegmentMK: Add index(es) to tar file backend
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Mon Mar 17 18:50:39 2014
@@ -57,6 +57,9 @@ public class FileStore implements Segmen
private static final String JOURNAL_FILE_NAME = "journal.log";
+ private static final boolean MEMORY_MAPPING_DEFAULT =
+ "64".equals(System.getProperty("sun.arch.data.model", "32"));
+
private final SegmentTracker tracker;
private final File directory;
@@ -106,6 +109,11 @@ public class FileStore implements Segmen
this(null, directory, maxFileSizeMB, memoryMapping);
}
+ public FileStore(File directory, int maxFileSizeMB)
+ throws IOException {
+ this(null, directory, maxFileSizeMB, MEMORY_MAPPING_DEFAULT);
+ }
+
public FileStore(File directory, int maxFileSizeMB, int cacheSizeMB,
boolean memoryMapping) throws IOException {
this(null, directory, EMPTY_NODE, maxFileSizeMB, cacheSizeMB,
memoryMapping);
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
Mon Mar 17 18:50:39 2014
@@ -18,7 +18,6 @@ package org.apache.jackrabbit.oak.plugin
import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
-import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
@@ -30,17 +29,16 @@ class MappedAccess implements FileAccess
private boolean updated = false;
- MappedAccess(File file, int length) throws IOException {
- RandomAccessFile f = new RandomAccessFile(file, "rw");
+ MappedAccess(RandomAccessFile file, int length) throws IOException {
try {
- long l = f.length();
+ long l = file.length();
if (l == 0) { // it's a new file
l = length;
updated = true;
}
- buffer = f.getChannel().map(READ_WRITE, 0, l);
+ buffer = file.getChannel().map(READ_WRITE, 0, l);
} finally {
- f.close();
+ file.close();
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
Mon Mar 17 18:50:39 2014
@@ -19,7 +19,6 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
-import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
@@ -30,8 +29,8 @@ class RandomAccess implements FileAccess
private final RandomAccessFile file;
- RandomAccess(@Nonnull File file) throws IOException {
- this.file = new RandomAccessFile(checkNotNull(file), "rw");
+ RandomAccess(@Nonnull RandomAccessFile file) throws IOException {
+ this.file = checkNotNull(file);
}
@Override
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
Mon Mar 17 18:50:39 2014
@@ -16,9 +16,6 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.file;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
class TarEntry {
private final int offset;
@@ -30,8 +27,12 @@ class TarEntry {
this.size = size;
}
- ByteBuffer read(FileAccess access) throws IOException {
- return access.read(offset, size);
+ int offset() {
+ return offset;
+ }
+
+ int size() {
+ return size;
}
}
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
Mon Mar 17 18:50:39 2014
@@ -19,12 +19,16 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Maps.newConcurrentMap;
+import static com.google.common.collect.Maps.newTreeMap;
import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
import java.util.UUID;
class TarFile {
@@ -42,6 +46,8 @@ class TarFile {
private final int maxFileSize;
+ private final byte[] indexEntryName;
+
private final Map<UUID, TarEntry> entries = newConcurrentMap();
TarFile(File file, int maxFileSize, boolean memoryMapping)
@@ -49,46 +55,35 @@ class TarFile {
long len = file.length();
checkState(len <= Integer.MAX_VALUE);
this.maxFileSize = Math.max((int) len, maxFileSize);
+ checkState(maxFileSize % BLOCK_SIZE == 0);
+ checkState(maxFileSize > 5 * BLOCK_SIZE);
+ this.indexEntryName = (file.getName() + ".idx").getBytes(UTF_8);
this.file = file;
+ RandomAccessFile f = new RandomAccessFile(file, "rw");
if (memoryMapping) {
- this.access = new MappedAccess(file, this.maxFileSize);
+ this.access = new MappedAccess(f, this.maxFileSize);
} else {
- this.access = new RandomAccess(file);
+ this.access = new RandomAccess(f);
}
+
+ this.position = 0;
if (len == 0) {
- // allocate the full file by writing the last two blocks
+ // allocate the full file by writing one big index entry
+ writeEntryHeader(indexEntryName, maxFileSize - 3 * BLOCK_SIZE);
+ // zero-out the last four bytes to indicate an empty index
+ access.write(
+ maxFileSize - (ZERO_BYTES.length * 2 + 4),
+ ZERO_BYTES, 0, 4);
+ // tar format expects the last two blocks to be zero
access.write(
maxFileSize - ZERO_BYTES.length * 2,
ZERO_BYTES, 0, ZERO_BYTES.length);
access.write(
maxFileSize - ZERO_BYTES.length,
ZERO_BYTES, 0, ZERO_BYTES.length);
- }
-
- this.position = 0;
- while (position + BLOCK_SIZE <= len) {
- // read the tar header block
- ByteBuffer buffer = this.access.read(position, BLOCK_SIZE);
- String name = readString(buffer, 100);
- buffer.position(124);
- int size = readNumber(buffer, 12);
- // TODO: verify the checksum, magic, etc.?
-
- if (name.isEmpty() && size == 0) {
- break; // no more entries in this file
- } else if (position + BLOCK_SIZE + size > len) {
- break; // invalid entry, truncate the file at this point
- }
-
- try {
- UUID id = UUID.fromString(name);
- entries.put(id, new TarEntry(position + BLOCK_SIZE, size));
- } catch (IllegalArgumentException e) {
- throw new IOException("Unexpected tar entry: " + name);
- }
-
- position += (1 + (size + BLOCK_SIZE - 1) / BLOCK_SIZE) *
BLOCK_SIZE;
+ } else {
+ readIndex(len);
}
}
@@ -99,7 +94,7 @@ class TarFile {
ByteBuffer readEntry(UUID uuid) throws IOException {
TarEntry entry = entries.get(uuid);
if (entry != null) {
- return entry.read(access);
+ return access.read(entry.offset(), entry.size());
} else {
return null;
}
@@ -107,15 +102,46 @@ class TarFile {
synchronized boolean writeEntry(
UUID uuid, byte[] b, int offset, int size) throws IOException {
- if (position + BLOCK_SIZE + size > maxFileSize) {
+ if (position + BLOCK_SIZE + size + 4 * BLOCK_SIZE > maxFileSize) {
+ writeEntryHeader(
+ indexEntryName, maxFileSize - 3 * BLOCK_SIZE - position);
+ ByteBuffer index = ByteBuffer.allocate(entries.size() * 24 + 4);
+ SortedMap<UUID, TarEntry> sorted = newTreeMap();
+ sorted.putAll(entries);
+ for (Map.Entry<UUID, TarEntry> entry : sorted.entrySet()) {
+ index.putLong(entry.getKey().getMostSignificantBits());
+ index.putLong(entry.getKey().getLeastSignificantBits());
+ index.putInt(entry.getValue().offset());
+ index.putInt(entry.getValue().size());
+ }
+ index.putInt(sorted.size());
+ access.write(
+ maxFileSize - 2 * BLOCK_SIZE - index.capacity(),
+ index.array(), 0, index.capacity());
return false;
}
+ writeEntryHeader(uuid.toString().getBytes(UTF_8), size);
+ position += BLOCK_SIZE;
+
+ access.write(position, b, offset, size);
+ entries.put(uuid, new TarEntry(position, size));
+ position += size;
+
+ int padding = BLOCK_SIZE - position % BLOCK_SIZE;
+ if (padding < BLOCK_SIZE) {
+ access.write(position, ZERO_BYTES, 0, padding);
+ position += padding;
+ }
+
+ return true;
+ }
+
+ protected void writeEntryHeader(byte[] name, int size) throws IOException {
byte[] header = new byte[BLOCK_SIZE];
// File name
- byte[] n = uuid.toString().getBytes(UTF_8);
- System.arraycopy(n, 0, header, 0, n.length);
+ System.arraycopy(name, 0, header, 0, name.length);
// File mode
System.arraycopy(
@@ -161,20 +187,8 @@ class TarFile {
header, 148, 6);
header[154] = 0;
+ checkState(position % BLOCK_SIZE == 0);
access.write(position, header, 0, BLOCK_SIZE);
- position += BLOCK_SIZE;
-
- access.write(position, b, offset, size);
- entries.put(uuid, new TarEntry(position, size));
- position += size;
-
- int padding = BLOCK_SIZE - position % BLOCK_SIZE;
- if (padding < BLOCK_SIZE) {
- access.write(position, ZERO_BYTES, 0, padding);
- position += padding;
- }
-
- return true;
}
public void flush() throws IOException {
@@ -187,6 +201,34 @@ class TarFile {
access.close();
}
+ private void readIndex(long len) throws IOException {
+ while (position + BLOCK_SIZE <= len) {
+ // read the tar header block
+ ByteBuffer buffer = this.access.read(position, BLOCK_SIZE);
+ String name = readString(buffer, 100);
+ buffer.position(124);
+ int size = readNumber(buffer, 12);
+ // TODO: verify the checksum, magic, etc.?
+
+ if (name.isEmpty() && size == 0) {
+ break; // no more entries in this file
+ } else if (Arrays.equals(name.getBytes(UTF_8), indexEntryName)) {
+ break; // index entry encountered, so stop here
+ } else if (position + BLOCK_SIZE + size > len) {
+ break; // invalid entry, truncate the file at this point
+ }
+
+ try {
+ UUID id = UUID.fromString(name);
+ entries.put(id, new TarEntry(position + BLOCK_SIZE, size));
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Unexpected tar entry: " + name);
+ }
+
+ position += (1 + (size + BLOCK_SIZE - 1) / BLOCK_SIZE) *
BLOCK_SIZE;
+ }
+ }
+
private static String readString(ByteBuffer buffer, int fieldSize) {
byte[] b = new byte[fieldSize];
buffer.get(b);
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1578510&r1=1578509&r2=1578510&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Mon Mar 17 18:50:39 2014
@@ -44,8 +44,8 @@ public class TarFileTest {
@Test
public void testOpenClose() throws IOException {
- new TarFile(file, 1024, true).close();
- new TarFile(file, 1024, false).close();
+ new TarFile(file, 10240, true).close();
+ new TarFile(file, 10240, false).close();
}
@Test
@@ -53,7 +53,7 @@ public class TarFileTest {
UUID id = UUID.randomUUID();
byte[] data = "Hello, World!".getBytes(UTF_8);
- TarFile tar = new TarFile(file, 1024, false);
+ TarFile tar = new TarFile(file, 10240, false);
try {
tar.writeEntry(id, data, 0, data.length);
assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
@@ -61,9 +61,9 @@ public class TarFileTest {
tar.close();
}
- assertEquals(1024, file.length());
+ assertEquals(10240, file.length());
- tar = new TarFile(file, 1024, false);
+ tar = new TarFile(file, 10240, false);
try {
assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
} finally {
