Author: jukka
Date: Wed Sep 18 19:52:47 2013
New Revision: 1524548
URL: http://svn.apache.org/r1524548
Log:
OAK-1001: SegmentMK: 32bit support for the file backend
Improved configurability of the different TarMK modes
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
Wed Sep 18 19:52:47 2013
@@ -30,7 +30,6 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.state.AbstractNodeStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStoreBranch;
public class SegmentNodeStore extends AbstractNodeStore {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
Wed Sep 18 19:52:47 2013
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Dictionary;
@@ -55,6 +56,12 @@ public class SegmentNodeStoreService ext
@Property(description="TarMK directory (if unset, use MongoDB)")
public static final String DIRECTORY = "repository.home";
+ @Property(description="TarMK mode (64 for memory mapping, 32 for normal
file access)")
+ public static final String MODE = "tarmk.mode";
+
+ @Property(description="TarMK maximum file size")
+ public static final String SIZE = "tarmk.size";
+
@Property(description="MongoDB host")
public static final String HOST = "host";
@@ -93,8 +100,25 @@ public class SegmentNodeStoreService ext
String host = lookup(context, HOST);
if (host == null) {
String directory = lookup(context, DIRECTORY);
+ if (directory == null) {
+ directory = "tarmk";
+ }
+
+ String mode = lookup(context, MODE);
+ if (mode == null) {
+ mode = System.getProperty(MODE,
+ System.getProperty("sun.arch.data.model", "32"));
+ }
+
+ String size = lookup(context, SIZE);
+ if (size == null) {
+ size = System.getProperty(SIZE, "268435456"); // 256MB
+ }
+
mongo = null;
- store = new FileStore(directory);
+ store = new FileStore(
+ new File(directory),
+ Integer.parseInt(size), "64".equals(mode));
} else {
int port = Integer.parseInt(String.valueOf(properties.get(PORT)));
String db = String.valueOf(properties.get(DB));
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Wed Sep 18 19:52:47 2013
@@ -42,7 +42,6 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
import org.apache.jackrabbit.oak.plugins.segment.Template;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@@ -55,12 +54,14 @@ public class FileStore implements Segmen
static final UUID JOURNALS_UUID = new UUID(0, 0);
- private static final int FILE_SIZE = 256 * 1024 * 1024;
-
private static final String FILE_NAME_FORMAT = "data%05d.tar";
private final File directory;
+ private final int maxFileSize;
+
+ private final boolean memoryMapping;
+
private final LinkedList<TarFile> files = newLinkedList();
private final Map<String, Journal> journals = newHashMap();
@@ -68,15 +69,18 @@ public class FileStore implements Segmen
private final Cache<UUID, Segment> segments =
CacheBuilder.newBuilder().maximumSize(1000).build();
- public FileStore(File directory, NodeState root) throws IOException {
+ public FileStore(File directory, int maxFileSize, boolean memoryMapping)
+ throws IOException {
checkNotNull(directory).mkdirs();
this.directory = directory;
+ this.maxFileSize = maxFileSize;
+ this.memoryMapping = memoryMapping;
for (int i = 0; true; i++) {
String name = String.format(FILE_NAME_FORMAT, i);
File file = new File(directory, name);
if (file.isFile()) {
- files.add(new TarFile(file, FILE_SIZE));
+ files.add(new TarFile(file, maxFileSize, memoryMapping));
} else {
break;
}
@@ -102,19 +106,11 @@ public class FileStore implements Segmen
if (!journals.containsKey("root")) {
NodeBuilder builder = EMPTY_NODE.builder();
- builder.setChildNode("root", root);
+ builder.setChildNode("root", EMPTY_NODE);
journals.put("root", new FileJournal(this,
builder.getNodeState()));
}
}
- public FileStore(File directory) throws IOException {
- this(directory, EMPTY_NODE);
- }
-
- public FileStore(String directory) throws IOException {
- this(new File(directory));
- }
-
public synchronized void close() {
for (TarFile file : files) {
try {
@@ -217,7 +213,8 @@ public class FileStore implements Segmen
if (files.isEmpty() || !files.getLast().writeEntry(
segmentId, buffer, 0, buffer.length)) {
String name = String.format(FILE_NAME_FORMAT, files.size());
- TarFile last = new TarFile(new File(directory, name), FILE_SIZE);
+ File file = new File(directory, name);
+ TarFile last = new TarFile(file, maxFileSize, memoryMapping);
checkState(last.writeEntry(segmentId, buffer, 0, buffer.length));
files.add(last);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
Wed Sep 18 19:52:47 2013
@@ -32,11 +32,6 @@ import com.google.common.collect.Immutab
class TarFile {
- private static boolean USE_MEMORY_MAPPING =
- System.getProperty("tarmk.mmap") != null
- ? Boolean.getBoolean("tarmk.mmap")
- : "64".equals(System.getProperty("sun.arch.data.model"));
-
/** The tar file block size. */
private static final int BLOCK_SIZE = 512;
@@ -59,17 +54,18 @@ class TarFile {
private int position = 0;
- private final int maxLength;
+ private final int maxFileSize;
private volatile Map<UUID, Location> entries;
- TarFile(File file, int maxLength) throws IOException {
+ TarFile(File file, int maxFileSize, boolean memoryMapping)
+ throws IOException {
long len = file.length();
checkState(len <= Integer.MAX_VALUE);
- this.maxLength = Math.max((int) len, maxLength);
+ this.maxFileSize = Math.max((int) len, maxFileSize);
- if (USE_MEMORY_MAPPING) {
- this.file = new MappedAccess(file, maxLength);
+ if (memoryMapping) {
+ this.file = new MappedAccess(file, this.maxFileSize);
} else {
this.file = new RandomAccess(file);
}
@@ -122,7 +118,7 @@ class TarFile {
synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size)
throws IOException {
- if (position + BLOCK_SIZE + size > maxLength) {
+ if (position + BLOCK_SIZE + size > maxFileSize) {
return false;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Wed Sep 18 19:52:47 2013
@@ -44,8 +44,8 @@ public class TarFileTest {
@Test
public void testOpenClose() throws IOException {
- TarFile tar = new TarFile(file, 1024);
- tar.close();
+ new TarFile(file, 1024, true).close();
+ new TarFile(file, 1024, false).close();
}
@Test
@@ -53,7 +53,7 @@ public class TarFileTest {
UUID id = UUID.randomUUID();
byte[] data = "Hello, World!".getBytes(UTF_8);
- TarFile tar = new TarFile(file, 1024);
+ TarFile tar = new TarFile(file, 1024, false);
try {
tar.writeEntry(id, data, 0, data.length);
assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
@@ -63,7 +63,7 @@ public class TarFileTest {
assertEquals(1024, file.length());
- tar = new TarFile(file, 1024);
+ tar = new TarFile(file, 1024, false);
try {
assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
} finally {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Wed Sep 18 19:52:47 2013
@@ -43,6 +43,9 @@ public class BenchmarkRunner {
.withRequiredArg().defaultsTo("localhost");
OptionSpec<Integer> port = parser.accepts("port", "MongoDB port")
.withRequiredArg().ofType(Integer.class).defaultsTo(27017);
+ OptionSpec<Boolean> mmap = parser.accepts("mmap", "TarMK memory
mapping")
+ .withOptionalArg().ofType(Boolean.class)
+
.defaultsTo("64".equals(System.getProperty("sun.arch.data.model")));
OptionSpec<Integer> cache = parser.accepts("cache", "cache size (MB)")
.withRequiredArg().ofType(Integer.class).defaultsTo(100);
OptionSpec<File> wikipedia =
@@ -68,7 +71,8 @@ public class BenchmarkRunner {
host.value(options), port.value(options), cacheSize *
MB),
OakRepositoryFixture.getSegment(
host.value(options), port.value(options), cacheSize *
MB),
- OakRepositoryFixture.getTar(base.value(options))
+ OakRepositoryFixture.getTar(
+ base.value(options), 256 * 1024 * 1024,
mmap.value(options))
};
Benchmark[] allBenchmarks = new Benchmark[] {
new LoginTest(),
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1524548&r1=1524547&r2=1524548&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
Wed Sep 18 19:52:47 2013
@@ -142,7 +142,9 @@ public abstract class OakRepositoryFixtu
};
}
- public static RepositoryFixture getTar(final File base) {
+ public static RepositoryFixture getTar(
+ final File base,
+ final int maxFileSize, final boolean memoryMapping) {
return new OakRepositoryFixture("Oak-Tar") {
private SegmentStore[] stores;
@Override
@@ -150,7 +152,8 @@ public abstract class OakRepositoryFixtu
Repository[] cluster = new Repository[n];
stores = new FileStore[cluster.length];
for (int i = 0; i < cluster.length; i++) {
- stores[i] = new FileStore(new File(base, unique));
+ stores[i] = new FileStore(
+ new File(base, unique), maxFileSize,
memoryMapping);
Oak oak = new Oak(new SegmentNodeStore(stores[i]));
cluster[i] = new Jcr(oak).createRepository();
}
