Author: jukka
Date: Tue Nov 26 19:09:08 2013
New Revision: 1545786
URL: http://svn.apache.org/r1545786
Log:
OAK-593: Segment-based MK
Automatically scale file and cache size parameters to MBs.
Use MongoClient instead of the deprecated Mongo class.
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakRepositoryStubBase.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakSegmentMKRepositoryStub.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakTarMKRepositoryStub.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
Tue Nov 26 19:09:08 2013
@@ -28,6 +28,8 @@ import com.google.common.cache.Cache;
public abstract class AbstractStore implements SegmentStore {
+ protected static final int MB = 1024 * 1024;
+
private final Cache<UUID, Segment> segments;
/**
@@ -40,11 +42,15 @@ public abstract class AbstractStore impl
private final SegmentWriter writer = new SegmentWriter(this);
- protected AbstractStore(int cacheSize) {
- this.segments = CacheLIRS.newBuilder()
- .weigher(Segment.WEIGHER)
- .maximumWeight(cacheSize)
- .build();
+ protected AbstractStore(int cacheSizeMB) {
+ if (cacheSizeMB > 0) {
+ this.segments = CacheLIRS.newBuilder()
+ .weigher(Segment.WEIGHER)
+ .maximumWeight(cacheSizeMB * MB)
+ .build();
+ } else {
+ this.segments = null;
+ }
}
protected abstract Segment loadSegment(UUID id) throws Exception;
@@ -56,9 +62,12 @@ public abstract class AbstractStore impl
@Override
public Segment readSegment(UUID id) {
- Segment segment = segments.getIfPresent(id);
- if (segment != null) {
- return segment;
+ Segment segment = null;
+ if (segments != null) {
+ segment = segments.getIfPresent(id);
+ if (segment != null) {
+ return segment;
+ }
}
segment = getWriter().getCurrentSegment(id);
@@ -66,6 +75,15 @@ public abstract class AbstractStore impl
return segment;
}
+ if (segments == null) {
+ // no caching, just load the segment directly
+ try {
+ return loadSegment(id);
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to load segment " +
id, e);
+ }
+ }
+
synchronized (this) {
segment = segments.getIfPresent(id);
while (segment == null && currentlyLoading.contains(id)) {
@@ -111,7 +129,9 @@ public abstract class AbstractStore impl
throw new RuntimeException("Interrupted", e);
}
}
- segments.invalidate(segmentId);
+ if (segments != null) {
+ segments.invalidate(segmentId);
+ }
}
@Override
@@ -140,7 +160,9 @@ public abstract class AbstractStore impl
}
}
records.invalidateAll();
- segments.invalidateAll();
+ if (segments != null) {
+ segments.invalidateAll();
+ }
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
Tue Nov 26 19:09:08 2013
@@ -27,7 +27,8 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import com.google.common.base.Preconditions;
-import com.mongodb.Mongo;
+import com.mongodb.MongoClient;
+
import org.apache.felix.scr.annotations.Activate;
import org.apache.felix.scr.annotations.Component;
import org.apache.felix.scr.annotations.ConfigurationPolicy;
@@ -60,7 +61,7 @@ public class SegmentNodeStoreService imp
@Property(description="TarMK mode (64 for memory mapping, 32 for normal
file access)")
public static final String MODE = "tarmk.mode";
- @Property(description="TarMK maximum file size")
+ @Property(description="TarMK maximum file size (MB)", intValue=256)
public static final String SIZE = "tarmk.size";
@Property(description="MongoDB host")
@@ -75,11 +76,9 @@ public class SegmentNodeStoreService imp
@Property(description="Cache size (MB)", intValue=200)
public static final String CACHE = "cache";
- private static final int MB = 1024 * 1024;
-
private String name;
- private Mongo mongo;
+ private MongoClient mongo;
private SegmentStore store;
@@ -111,7 +110,7 @@ public class SegmentNodeStoreService imp
String size = lookup(context, SIZE);
if (size == null) {
- size = System.getProperty(SIZE, "268435456"); // 256MB
+ size = System.getProperty(SIZE, "256");
}
mongo = null;
@@ -123,8 +122,8 @@ public class SegmentNodeStoreService imp
String db = String.valueOf(properties.get(DB));
int cache =
Integer.parseInt(String.valueOf(properties.get(CACHE)));
- mongo = new Mongo(host, port);
- store = new MongoStore(mongo.getDB(db), cache * MB);
+ mongo = new MongoClient(host, port);
+ store = new MongoStore(mongo.getDB(db), cache);
}
delegate = new SegmentNodeStore(store);
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Tue Nov 26 19:09:08 2013
@@ -42,7 +42,7 @@ import org.apache.jackrabbit.oak.spi.sta
public class FileStore extends AbstractStore {
- private static final int DEFAULT_MEMORY_CACHE_SIZE = 1 << 28; // 256MB
+ private static final int DEFAULT_MEMORY_CACHE_SIZE = 256;
private static final long JOURNAL_MAGIC = 0xdf36544212c0cb24L;
@@ -62,24 +62,24 @@ public class FileStore extends AbstractS
private final Map<String, Journal> journals = newHashMap();
- public FileStore(File directory, int maxFileSize, boolean memoryMapping)
+ public FileStore(File directory, int maxFileSizeMB, boolean memoryMapping)
throws IOException {
- this(directory, maxFileSize, DEFAULT_MEMORY_CACHE_SIZE, memoryMapping);
+ this(directory, maxFileSizeMB, DEFAULT_MEMORY_CACHE_SIZE,
memoryMapping);
}
- public FileStore(File directory, int maxFileSize, int cacheSize,
+ public FileStore(File directory, int maxFileSizeMB, int cacheSizeMB,
boolean memoryMapping) throws IOException {
- super(cacheSize);
+ super(cacheSizeMB);
checkNotNull(directory).mkdirs();
this.directory = directory;
- this.maxFileSize = maxFileSize;
+ this.maxFileSize = maxFileSizeMB * MB;
this.memoryMapping = memoryMapping;
for (int i = 0; true; i++) {
String name = String.format(FILE_NAME_FORMAT, "bulk", i);
File file = new File(directory, name);
if (file.isFile()) {
- bulkFiles.add(new TarFile(file, maxFileSize, memoryMapping));
+ bulkFiles.add(new TarFile(file, maxFileSizeMB, memoryMapping));
} else {
break;
}
@@ -89,7 +89,7 @@ public class FileStore extends AbstractS
String name = String.format(FILE_NAME_FORMAT, "data", i);
File file = new File(directory, name);
if (file.isFile()) {
- dataFiles.add(new TarFile(file, maxFileSize, memoryMapping));
+ dataFiles.add(new TarFile(file, maxFileSizeMB, memoryMapping));
} else {
break;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java
Tue Nov 26 19:09:08 2013
@@ -39,7 +39,7 @@ public class MemoryStore extends Abstrac
Maps.newConcurrentMap();
public MemoryStore(NodeState root) {
- super(1024 * 1024);
+ super(0);
NodeBuilder builder = EMPTY_NODE.builder();
builder.setChildNode("root", root);
journals.put("root", new MemoryJournal(this, builder.getNodeState()));
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/mongo/MongoStore.java
Tue Nov 26 19:09:08 2013
@@ -36,7 +36,6 @@ import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
-import com.mongodb.Mongo;
import com.mongodb.WriteConcern;
public class MongoStore extends AbstractStore {
@@ -49,8 +48,8 @@ public class MongoStore extends Abstract
private final Map<String, Journal> journals = Maps.newHashMap();
- public MongoStore(DB db, int cacheSize) {
- super(cacheSize);
+ public MongoStore(DB db, int cacheSizeMB) {
+ super(cacheSizeMB);
this.db = checkNotNull(db);
this.segments = db.getCollection("segments");
NodeBuilder builder = EMPTY_NODE.builder();
@@ -60,10 +59,6 @@ public class MongoStore extends Abstract
concern, builder.getNodeState()));
}
- public MongoStore(Mongo mongo, int cacheSize) {
- this(mongo.getDB("Oak"), cacheSize);
- }
-
@Override
public void close() {
}
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakRepositoryStubBase.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakRepositoryStubBase.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakRepositoryStubBase.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakRepositoryStubBase.java
Tue Nov 26 19:09:08 2013
@@ -38,8 +38,8 @@ import org.apache.jackrabbit.test.Reposi
public class OakRepositoryStubBase extends RepositoryStub {
- private static final int MAX_FILE_SIZE = 64 * 1024 * 1024;
- private static final int CACHE_SIZE = 32 * 1024 * 1024;
+ private static final int MAX_FILE_SIZE = 64;
+ private static final int CACHE_SIZE = 32;
private static final boolean MMAP =
System.getProperty("sun.arch.data.model", "32").equals("64");
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakSegmentMKRepositoryStub.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakSegmentMKRepositoryStub.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakSegmentMKRepositoryStub.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakSegmentMKRepositoryStub.java
Tue Nov 26 19:09:08 2013
@@ -26,7 +26,8 @@ import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.UnsupportedRepositoryOperationException;
-import com.mongodb.Mongo;
+import com.mongodb.MongoClient;
+
import org.apache.jackrabbit.api.JackrabbitSession;
import org.apache.jackrabbit.api.security.principal.PrincipalIterator;
import org.apache.jackrabbit.api.security.principal.PrincipalManager;
@@ -50,7 +51,7 @@ public class OakSegmentMKRepositoryStub
protected static final String DB =
System.getProperty("segment.db", "SegmentMK");
- private final Mongo connection;
+ private final MongoClient connection;
private final Repository repository;
@@ -65,9 +66,9 @@ public class OakSegmentMKRepositoryStub
Session session = null;
try {
- this.connection = new Mongo(HOST, PORT);
+ this.connection = new MongoClient(HOST, PORT);
Jcr jcr = new Jcr(new Oak(new SegmentNodeStore(
- new MongoStore(connection.getDB(DB), 4 * 1024 * 1024))));
+ new MongoStore(connection.getDB(DB), 4))));
this.repository = jcr.createRepository();
session = getRepository().login(superuser);
@@ -90,7 +91,7 @@ public class OakSegmentMKRepositoryStub
public static boolean isAvailable() {
try {
- Mongo mongo = new Mongo(HOST, PORT);
+ MongoClient mongo = new MongoClient(HOST, PORT);
try {
mongo.getDatabaseNames();
return true;
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakTarMKRepositoryStub.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakTarMKRepositoryStub.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakTarMKRepositoryStub.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/OakTarMKRepositoryStub.java
Tue Nov 26 19:09:08 2013
@@ -58,7 +58,7 @@ public class OakTarMKRepositoryStub exte
try {
File directory =
new File("target", "tarmk-" + System.currentTimeMillis());
- this.store = new FileStore(directory, 1024 * 1024, false);
+ this.store = new FileStore(directory, 1, false);
Jcr jcr = new Jcr(new Oak(new SegmentNodeStore(store)));
this.repository = jcr.createRepository();
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
Tue Nov 26 19:09:08 2013
@@ -90,9 +90,9 @@ public class BenchmarkRunner {
dbName.value(options), dropDBAfterTest.value(options),
cacheSize * MB),
OakRepositoryFixture.getSegment(
- host.value(options), port.value(options), cacheSize *
MB),
+ host.value(options), port.value(options), cacheSize),
OakRepositoryFixture.getTar(
- base.value(options), 256 * MB, cacheSize * MB,
mmap.value(options))
+ base.value(options), 256, cacheSize,
mmap.value(options))
};
Benchmark[] allBenchmarks = new Benchmark[] {
new LoginTest(),
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakRepositoryFixture.java
Tue Nov 26 19:09:08 2013
@@ -34,7 +34,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.plugins.segment.mongo.MongoStore;
-import com.mongodb.Mongo;
+import com.mongodb.MongoClient;
public abstract class OakRepositoryFixture implements RepositoryFixture {
@@ -161,17 +161,17 @@ public abstract class OakRepositoryFixtu
}
public static RepositoryFixture getSegment(
- final String host, final int port, final int cacheSize) {
+ final String host, final int port, final int cacheSizeMB) {
return new OakRepositoryFixture("Oak-Segment") {
private SegmentStore[] stores;
- private Mongo mongo;
+ private MongoClient mongo;
@Override
public Repository[] setUpCluster(int n) throws Exception {
Repository[] cluster = new Repository[n];
stores = new SegmentStore[cluster.length];
- mongo = new Mongo(host, port);
+ mongo = new MongoClient(host, port);
for (int i = 0; i < cluster.length; i++) {
- stores[i] = new MongoStore(mongo.getDB(unique), cacheSize);
+ stores[i] = new MongoStore(mongo.getDB(unique),
cacheSizeMB);
Oak oak = new Oak(new SegmentNodeStore(stores[i]));
cluster[i] = new Jcr(oak).createRepository();
}
@@ -189,8 +189,8 @@ public abstract class OakRepositoryFixtu
}
public static RepositoryFixture getTar(
- final File base,
- final int maxFileSize, final int cacheSize, final boolean
memoryMapping) {
+ final File base, final int maxFileSizeMB, final int cacheSizeMB,
+ final boolean memoryMapping) {
return new OakRepositoryFixture("Oak-Tar") {
private SegmentStore[] stores;
@Override
@@ -199,7 +199,8 @@ public abstract class OakRepositoryFixtu
stores = new FileStore[cluster.length];
for (int i = 0; i < cluster.length; i++) {
stores[i] = new FileStore(
- new File(base, unique), maxFileSize, cacheSize,
memoryMapping);
+ new File(base, unique),
+ maxFileSizeMB, cacheSizeMB, memoryMapping);
Oak oak = new Oak(new SegmentNodeStore(stores[i]));
cluster[i] = new Jcr(oak).createRepository();
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1545786&r1=1545785&r2=1545786&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Tue Nov 26 19:09:08 2013
@@ -55,7 +55,6 @@ import org.eclipse.jetty.servlet.Servlet
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
-import com.google.common.collect.Sets;
public class Main {
@@ -95,7 +94,7 @@ public class Main {
} else {
System.out.println("TarMK " + args[0]);
File file = new File(args[0]);
- FileStore store = new FileStore(file, 256 * 1024 * 1024,
false);
+ FileStore store = new FileStore(file, 256, false);
try {
if (args.length == 1) {
Map<UUID, List<UUID>> idmap = Maps.newHashMap();
@@ -173,8 +172,7 @@ public class Main {
RepositoryContext source = RepositoryContext.create(
RepositoryConfig.create(new File(olddir)));
try {
- FileStore store = new FileStore(
- new File(newdir), 256 * 1024 * 1024, true);
+ FileStore store = new FileStore(new File(newdir), 256, true);
try {
NodeStore target = new SegmentNodeStore(store);
new RepositoryUpgrade(source, target).copy();
