Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Template.java Thu Mar 13 14:31:57 2014 @@ -159,10 +159,9 @@ public class Template { return childName; } - SegmentPropertyState getProperty( - Segment segment, RecordId recordId, int index) { + SegmentPropertyState getProperty(RecordId recordId, int index) { checkElementIndex(index, properties.length); - segment = checkNotNull(segment).getSegment(checkNotNull(recordId)); + Segment segment = checkNotNull(recordId).getSegment(); int offset = recordId.getOffset() + RECORD_ID_BYTES; if (childName != ZERO_CHILD_NODES) { @@ -170,23 +169,22 @@ public class Template { } offset += index * RECORD_ID_BYTES; return new SegmentPropertyState( - segment, segment.readRecordId(offset), properties[index]); + segment.readRecordId(offset), properties[index]); } - MapRecord getChildNodeMap(Segment segment, RecordId recordId) { + MapRecord getChildNodeMap(RecordId recordId) { checkState(childName != ZERO_CHILD_NODES); - segment = segment.getSegment(recordId); + Segment segment = recordId.getSegment(); int offset = recordId.getOffset() + RECORD_ID_BYTES; RecordId childNodesId = segment.readRecordId(offset); return segment.readMap(childNodesId); } - public NodeState getChildNode( - String name, Segment segment, RecordId recordId) { + public NodeState getChildNode(String name, RecordId recordId) { if (childName == ZERO_CHILD_NODES) { return MISSING_NODE; } else if (childName == MANY_CHILD_NODES) { - MapRecord map = getChildNodeMap(segment, recordId); + MapRecord map = getChildNodeMap(recordId); MapEntry child = map.getEntry(name); if (child != null) { return child.getNodeState(); @@ -194,40 +192,38 @@ public class Template { return MISSING_NODE; } } else if (name.equals(childName)) { - segment = segment.getSegment(recordId); + Segment segment = recordId.getSegment(); int offset = recordId.getOffset() + RECORD_ID_BYTES; RecordId childNodeId = segment.readRecordId(offset); - return new SegmentNodeState(segment, childNodeId); + return new SegmentNodeState(childNodeId); } else { return MISSING_NODE; } } - Iterable<? extends ChildNodeEntry> getChildNodeEntries( - Segment segment, RecordId recordId) { + Iterable<? extends ChildNodeEntry> getChildNodeEntries(RecordId recordId) { if (childName == ZERO_CHILD_NODES) { return Collections.emptyList(); } else if (childName == MANY_CHILD_NODES) { - MapRecord map = getChildNodeMap(segment, recordId); + MapRecord map = getChildNodeMap(recordId); return map.getEntries(); } else { - segment = segment.getSegment(recordId); + Segment segment = recordId.getSegment(); int offset = recordId.getOffset() + RECORD_ID_BYTES; RecordId childNodeId = segment.readRecordId(offset); return Collections.singletonList(new MemoryChildNodeEntry( - childName, new SegmentNodeState(segment, childNodeId))); + childName, new SegmentNodeState(childNodeId))); } } - public boolean compare( - Segment thisSegment, RecordId thisId, Segment thatSegment, RecordId thatId) { + public boolean compare(RecordId thisId, RecordId thatId) { checkNotNull(thisId); checkNotNull(thatId); // Compare properties for (int i = 0; i < properties.length; i++) { - PropertyState thisProperty = getProperty(thisSegment, thisId, i); - PropertyState thatProperty = getProperty(thatSegment, thatId, i); + PropertyState thisProperty = getProperty(thisId, i); + PropertyState thatProperty = getProperty(thatId, i); if (!thisProperty.equals(thatProperty)) { return false; } @@ -237,13 +233,13 @@ public class Template { if (childName == ZERO_CHILD_NODES) { return true; } else if (childName != MANY_CHILD_NODES) { - NodeState thisChild = getChildNode(childName, thisSegment, thisId); - NodeState thatChild = getChildNode(childName, thatSegment, thatId); + NodeState thisChild = getChildNode(childName, thisId); + NodeState thatChild = getChildNode(childName, thatId); return thisChild.equals(thatChild); } else { // TODO: Leverage the HAMT data structure for the comparison - MapRecord thisMap = getChildNodeMap(thisSegment, thisId); - MapRecord thatMap = getChildNodeMap(thatSegment, thatId); + MapRecord thisMap = getChildNodeMap(thisId); + MapRecord thatMap = getChildNodeMap(thatId); if (fastEquals(thisMap, thatMap)) { return true; // shortcut } else if (thisMap.size() != thatMap.size()) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Thu Mar 13 14:31:57 2014 @@ -20,25 +20,22 @@ import static com.google.common.base.Pre import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newCopyOnWriteArrayList; -import static com.google.common.collect.Sets.newHashSet; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.isBulkSegmentId; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.cache.CacheLIRS; import org.apache.jackrabbit.oak.plugins.segment.RecordId; import org.apache.jackrabbit.oak.plugins.segment.Segment; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; import org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; @@ -48,8 +45,6 @@ import org.apache.jackrabbit.oak.spi.sta import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.cache.Cache; - public class FileStore implements SegmentStore { private static final Logger log = LoggerFactory.getLogger(FileStore.class); @@ -62,22 +57,9 @@ public class FileStore implements Segmen private static final String JOURNAL_FILE_NAME = "journal.log"; - private final SegmentIdFactory factory = new SegmentIdFactory(); - - private final SegmentWriter writer = new SegmentWriter(this, factory); + private final SegmentIdFactory factory = new SegmentIdFactory(this); - protected final Cache<UUID, Segment> segments; - - /** - * Identifiers of the segments that are currently being loaded. - */ - private final Set<UUID> currentlyLoading = newHashSet(); - - /** - * Number of threads that are currently waiting for segments to be loaded. - * Used to avoid extra {@link #notifyAll()} calls when nobody is waiting. - */ - private int currentlyWaiting = 0; + private final SegmentWriter writer = new SegmentWriter(this); private final File directory; @@ -132,16 +114,6 @@ public class FileStore implements Segmen this.maxFileSize = maxFileSizeMB * MB; this.memoryMapping = memoryMapping; - if (memoryMapping) { - // let the OS take care of caching - this.segments = null; - } else { - this.segments = CacheLIRS.newBuilder() - .weigher(Segment.WEIGHER) - .maximumWeight(cacheSizeMB * MB) - .build(); - } - for (int i = 0; true; i++) { String name = String.format(FILE_NAME_FORMAT, "bulk", i); File file = new File(directory, name); @@ -170,7 +142,7 @@ public class FileStore implements Segmen while (line != null) { int space = line.indexOf(' '); if (space != -1) { - id = RecordId.fromString(line.substring(0, space)); + id = RecordId.fromString(factory, line.substring(0, space)); } line = journalFile.readLine(); } @@ -233,33 +205,40 @@ public class FileStore implements Segmen } } } - - if (segments != null) { - for (Segment segment : segments.asMap().values().toArray(new Segment[0])) { - segment.dropOldCacheEntries(); - } - } } - public Iterable<UUID> getSegmentIds() { - List<UUID> ids = newArrayList(); + public Iterable<SegmentId> getSegmentIds() { + List<SegmentId> ids = newArrayList(); for (TarFile file : dataFiles) { - ids.addAll(file.getUUIDs()); + for (UUID uuid : file.getUUIDs()) { + ids.add(factory.getSegmentId( + uuid.getMostSignificantBits(), + uuid.getLeastSignificantBits())); + } } for (TarFile file : bulkFiles) { - ids.addAll(file.getUUIDs()); + for (UUID uuid : file.getUUIDs()) { + ids.add(factory.getSegmentId( + uuid.getMostSignificantBits(), + uuid.getLeastSignificantBits())); + } } return ids; } @Override + public SegmentIdFactory getFactory() { + return factory; + } + + @Override public SegmentWriter getWriter() { return writer; } @Override public SegmentNodeState getHead() { - return new SegmentNodeState(getWriter().getDummySegment(), head.get()); + return new SegmentNodeState(head.get()); } @Override @@ -282,19 +261,6 @@ public class FileStore implements Segmen } synchronized (this) { - if (segments != null) { - synchronized (segments) { - while (!currentlyLoading.isEmpty()) { - try { - segments.wait(); // for concurrent loads to finish - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted", e); - } - } - segments.invalidateAll(); - } - } - flush(); journalFile.close(); @@ -317,67 +283,56 @@ public class FileStore implements Segmen } @Override - public Segment readSegment(UUID id) { - if (isBulkSegmentId(id)) { - return loadSegment(id, bulkFiles); - } - - Segment segment = getWriter().getCurrentSegment(id); - if (segment != null) { - return segment; + public boolean containsSegment(SegmentId id) { + if (id.getStore() == this) { + return true; + } else if (id.isDataSegmentId()) { + return containsSegment(id, dataFiles); + } else { + return containsSegment(id, bulkFiles); } + } - if (segments == null) { - // no in-memory cache, load the segment directly - return loadSegment(id, dataFiles); + @Override + public Segment readSegment(SegmentId id) { + if (id.isBulkSegmentId()) { + return loadSegment(id, bulkFiles); } - synchronized (segments) { - // check if the segment is already cached - segment = segments.getIfPresent(id); - // ... or currently being loaded - while (segment == null && currentlyLoading.contains(id)) { - currentlyWaiting++; - try { - segments.wait(); // for another thread to load the segment - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted", e); - } finally { - currentlyWaiting--; - } - segment = segments.getIfPresent(id); - } - if (segment != null) { - // found the segment in the cache - return segment; - } - // not yet cached, so start let others know that we're loading it - currentlyLoading.add(id); + Segment segment = getWriter().getCurrentSegment(id); + if (segment == null) { + segment = loadSegment(id, dataFiles); } + return segment; + } - try { - segment = loadSegment(id, dataFiles); - } finally { - synchronized (segments) { - if (segment != null) { - segments.put(id, segment); - } - currentlyLoading.remove(id); - if (currentlyWaiting > 0) { - segments.notifyAll(); + private boolean containsSegment(SegmentId id, List<TarFile> files) { + UUID uuid = new UUID( + id.getMostSignificantBits(), + id.getLeastSignificantBits()); + for (TarFile file : files) { + try { + ByteBuffer buffer = file.readEntry(uuid); + if (buffer != null) { + return true; } + } catch (IOException e) { + throw new RuntimeException( + "Failed to access file " + file, e); } } - - return segment; + return false; } - protected Segment loadSegment(UUID id, List<TarFile> files) { + private Segment loadSegment(SegmentId id, List<TarFile> files) { + UUID uuid = new UUID( + id.getMostSignificantBits(), + id.getLeastSignificantBits()); for (TarFile file : files) { try { - ByteBuffer buffer = file.readEntry(id); + ByteBuffer buffer = file.readEntry(uuid); if (buffer != null) { - return new Segment(this, factory, id, buffer); + return new Segment(this, id, buffer); } } catch (IOException e) { throw new RuntimeException( @@ -390,22 +345,25 @@ public class FileStore implements Segmen @Override public synchronized void writeSegment( - UUID segmentId, byte[] data, int offset, int length) { + SegmentId id, byte[] data, int offset, int length) { // select whether to write a data or a bulk segment List<TarFile> files = dataFiles; String base = "data"; - if (isBulkSegmentId(segmentId)) { + if (id.isBulkSegmentId()) { files = bulkFiles; base = "bulk"; } try { + UUID uuid = new UUID( + id.getMostSignificantBits(), + id.getLeastSignificantBits()); if (files.isEmpty() || !files.get(files.size() - 1).writeEntry( - segmentId, data, offset, length)) { + uuid, data, offset, length)) { String name = String.format(FILE_NAME_FORMAT, base, files.size()); File file = new File(directory, name); TarFile last = new TarFile(file, maxFileSize, memoryMapping); - checkState(last.writeEntry(segmentId, data, offset, length)); + checkState(last.writeEntry(uuid, data, offset, length)); files.add(last); } } catch (IOException e) { Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java Thu Mar 13 14:31:57 2014 @@ -96,8 +96,8 @@ class TarFile { return entries.keySet(); } - ByteBuffer readEntry(UUID id) throws IOException { - TarEntry entry = entries.get(id); + ByteBuffer readEntry(UUID uuid) throws IOException { + TarEntry entry = entries.get(uuid); if (entry != null) { return entry.read(access); } else { @@ -105,8 +105,8 @@ class TarFile { } } - synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size) - throws IOException { + synchronized boolean writeEntry( + UUID uuid, byte[] b, int offset, int size) throws IOException { if (position + BLOCK_SIZE + size > maxFileSize) { return false; } @@ -114,7 +114,7 @@ class TarFile { byte[] header = new byte[BLOCK_SIZE]; // File name - byte[] n = id.toString().getBytes(UTF_8); + byte[] n = uuid.toString().getBytes(UTF_8); System.arraycopy(n, 0, header, 0, n.length); // File mode @@ -165,7 +165,7 @@ class TarFile { position += BLOCK_SIZE; access.write(position, b, offset, size); - entries.put(id, new TarEntry(position, size)); + entries.put(uuid, new TarEntry(position, size)); position += size; int padding = BLOCK_SIZE - position % BLOCK_SIZE; Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/http/HttpStore.java Thu Mar 13 14:31:57 2014 @@ -17,8 +17,6 @@ package org.apache.jackrabbit.oak.plugins.segment.http; import static com.google.common.base.Charsets.UTF_8; -import static com.google.common.collect.Sets.newHashSet; -import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.isBulkSegmentId; import java.io.BufferedReader; import java.io.IOException; @@ -29,58 +27,40 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; -import java.util.Set; -import java.util.UUID; import javax.annotation.CheckForNull; import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.cache.CacheLIRS; import org.apache.jackrabbit.oak.plugins.segment.RecordId; import org.apache.jackrabbit.oak.plugins.segment.Segment; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; import org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; import org.apache.jackrabbit.oak.plugins.segment.SegmentWriter; -import com.google.common.cache.Cache; import com.google.common.io.ByteStreams; public class HttpStore implements SegmentStore { - protected static final int MB = 1024 * 1024; + private final SegmentIdFactory factory = new SegmentIdFactory(this); - private final SegmentIdFactory factory = new SegmentIdFactory(); - - private final SegmentWriter writer = new SegmentWriter(this, factory); + private final SegmentWriter writer = new SegmentWriter(this); private final URL base; - private final Cache<UUID, Segment> segments; - - /** - * Identifiers of the segments that are currently being loaded. - */ - private final Set<UUID> currentlyLoading = newHashSet(); - - /** - * Number of threads that are currently waiting for segments to be loaded. - * Used to avoid extra {@link #notifyAll()} calls when nobody is waiting. - */ - private int currentlyWaiting = 0; - /** * @param base * make sure the url ends with a slash "/", otherwise the * requests will end up as absolute instead of relative - * @param cacheSizeMB */ - public HttpStore(URL base, int cacheSizeMB) { + public HttpStore(URL base) { this.base = base; - this.segments = CacheLIRS.newBuilder() - .weigher(Segment.WEIGHER) - .maximumWeight(cacheSizeMB * MB) - .build(); + } + + @Override + public SegmentIdFactory getFactory() { + return factory; } @Override @@ -97,8 +77,7 @@ public class HttpStore implements Segmen BufferedReader reader = new BufferedReader( new InputStreamReader(stream, UTF_8)); return new SegmentNodeState( - getWriter().getDummySegment(), - RecordId.fromString(reader.readLine())); + RecordId.fromString(factory, reader.readLine())); } finally { stream.close(); } @@ -118,64 +97,32 @@ public class HttpStore implements Segmen } @Override - public Segment readSegment(UUID id) { - if (isBulkSegmentId(id)) { + public boolean containsSegment(SegmentId id) { + return id.getStore() == this + || loadSegment(id) != null; + } + + @Override + public Segment readSegment(SegmentId id) { + if (id.isBulkSegmentId()) { return loadSegment(id); } Segment segment = getWriter().getCurrentSegment(id); - if (segment != null) { - return segment; - } - - synchronized (segments) { - // check if the segment is already cached - segment = segments.getIfPresent(id); - // ... or currently being loaded - while (segment == null && currentlyLoading.contains(id)) { - currentlyWaiting++; - try { - segments.wait(); // for another thread to load the segment - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted", e); - } finally { - currentlyWaiting--; - } - segment = segments.getIfPresent(id); - } - if (segment != null) { - // found the segment in the cache - return segment; - } - // not yet cached, so start let others know that we're loading it - currentlyLoading.add(id); - } - - try { + if (segment == null) { segment = loadSegment(id); - } finally { - synchronized (segments) { - if (segment != null) { - segments.put(id, segment); - } - currentlyLoading.remove(id); - if (currentlyWaiting > 0) { - segments.notifyAll(); - } - } } - return segment; } - private Segment loadSegment(UUID uuid) { + private Segment loadSegment(SegmentId id) { try { URLConnection connection = - new URL(base, uuid.toString()).openConnection(); + new URL(base, id.toString()).openConnection(); InputStream stream = connection.getInputStream(); try { byte[] data = ByteStreams.toByteArray(stream); - return new Segment(this, factory, uuid, ByteBuffer.wrap(data)); + return new Segment(this, id, ByteBuffer.wrap(data)); } finally { stream.close(); } @@ -187,11 +134,11 @@ public class HttpStore implements Segmen } @Override - public void writeSegment(UUID segmentId, byte[] bytes, int offset, - int length) { + public void writeSegment( + SegmentId id, byte[] bytes, int offset, int length) { try { URLConnection connection = - new URL(base, segmentId.toString()).openConnection(); + new URL(base, id.toString()).openConnection(); connection.setDoInput(false); connection.setDoOutput(true); OutputStream stream = connection.getOutputStream(); @@ -209,16 +156,6 @@ public class HttpStore implements Segmen @Override public void close() { - synchronized (segments) { - while (!currentlyLoading.isEmpty()) { - try { - segments.wait(); // for concurrent loads to finish - } catch (InterruptedException e) { - throw new RuntimeException("Interrupted", e); - } - } - segments.invalidateAll(); - } } @Override @CheckForNull Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java (original) +++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/memory/MemoryStore.java Thu Mar 13 14:31:57 2014 @@ -19,13 +19,13 @@ package org.apache.jackrabbit.oak.plugin import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import java.nio.ByteBuffer; -import java.util.UUID; import java.util.concurrent.ConcurrentMap; import javax.annotation.Nonnull; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.plugins.segment.Segment; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; import org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; @@ -37,13 +37,13 @@ import com.google.common.collect.Maps; public class MemoryStore implements SegmentStore { - private final SegmentIdFactory factory = new SegmentIdFactory(); + private final SegmentIdFactory factory = new SegmentIdFactory(this); - private final SegmentWriter writer = new SegmentWriter(this, factory); + private final SegmentWriter writer = new SegmentWriter(this); private SegmentNodeState head; - private final ConcurrentMap<UUID, Segment> segments = + private final ConcurrentMap<SegmentId, Segment> segments = Maps.newConcurrentMap(); public MemoryStore(NodeState root) { @@ -60,6 +60,11 @@ public class MemoryStore implements Segm } @Override + public SegmentIdFactory getFactory() { + return factory; + } + + @Override public SegmentWriter getWriter() { return writer; } @@ -79,28 +84,33 @@ public class MemoryStore implements Segm } } + @Override + public boolean containsSegment(SegmentId id) { + return id.getStore() == this; + } + @Override @Nonnull - public Segment readSegment(UUID uuid) { - Segment segment = writer.getCurrentSegment(uuid); + public Segment readSegment(SegmentId id) { + Segment segment = writer.getCurrentSegment(id); if (segment == null) { - segment = segments.get(uuid); + segment = segments.get(id); } if (segment != null) { return segment; } else { - throw new IllegalArgumentException("Segment not found: " + uuid); + throw new IllegalArgumentException("Segment not found: " + id); } } @Override public void writeSegment( - UUID segmentId, byte[] data, int offset, int length) { + SegmentId id, byte[] data, int offset, int length) { ByteBuffer buffer = ByteBuffer.allocate(length); buffer.put(data, offset, length); buffer.rewind(); - Segment segment = new Segment(this, factory, segmentId, buffer); - if (segments.putIfAbsent(segmentId, segment) != null) { - throw new IllegalStateException("Segment override: " + segmentId); + Segment segment = new Segment(this, id, buffer); + if (segments.putIfAbsent(id, segment) != null) { + throw new IllegalStateException("Segment override: " + id); } } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointTest.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CheckpointTest.java Thu Mar 13 14:31:57 2014 @@ -42,7 +42,7 @@ public class CheckpointTest { verifyNS(store, false); // gc? - store.retrieve(new SegmentIdFactory().newDataSegmentId().toString()); + store.retrieve("missing-checkpoint"); } private static void verifyNS(SegmentNodeStore store, boolean exists) { Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordTest.java Thu Mar 13 14:31:57 2014 @@ -61,8 +61,7 @@ public class RecordTest { @Test public void testBlockRecord() { RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); - BlockRecord block = new BlockRecord( - writer.getDummySegment(), blockId, bytes.length); + BlockRecord block = new BlockRecord(blockId, bytes.length); // Check reading with all valid positions and lengths for (int n = 1; n < bytes.length; n++) { @@ -119,13 +118,12 @@ public class RecordTest { private ListRecord writeList(int size, RecordId id) { List<RecordId> list = Collections.nCopies(size, id); - return new ListRecord( - writer.getDummySegment(), writer.writeList(list), size); + return new ListRecord(writer.writeList(list), size); } @Test public void testListWithLotsOfReferences() { // OAK-1184 - SegmentIdFactory factory = new SegmentIdFactory(); + SegmentIdFactory factory = store.getFactory(); List<RecordId> list = newArrayList(); for (int i = 0; i < 1000; i++) { list.add(new RecordId(factory.newBulkSegmentId(), 0)); Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentIdFactoryTest.java Thu Mar 13 14:31:57 2014 @@ -18,25 +18,24 @@ package org.apache.jackrabbit.oak.plugin import static junit.framework.Assert.assertFalse; import static junit.framework.Assert.assertTrue; -import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.isBulkSegmentId; -import static org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory.isDataSegmentId; import java.util.Set; import java.util.UUID; +import org.apache.jackrabbit.oak.plugins.segment.memory.MemoryStore; import org.junit.Test; public class SegmentIdFactoryTest { - private final SegmentIdFactory factory = new SegmentIdFactory(); + private final SegmentIdFactory factory = new MemoryStore().getFactory(); @Test public void segmentIdType() { - assertTrue(isDataSegmentId(factory.newDataSegmentId())); - assertTrue(isBulkSegmentId(factory.newBulkSegmentId())); + assertTrue(factory.newDataSegmentId().isDataSegmentId()); + assertTrue(factory.newBulkSegmentId().isBulkSegmentId()); - assertFalse(isBulkSegmentId(factory.newDataSegmentId())); - assertFalse(isDataSegmentId(factory.newBulkSegmentId())); + assertFalse(factory.newDataSegmentId().isBulkSegmentId()); + assertFalse(factory.newBulkSegmentId().isDataSegmentId()); } @Test @@ -48,11 +47,11 @@ public class SegmentIdFactoryTest { @Test public void referencedSegmentIds() throws InterruptedException { - UUID a = factory.newDataSegmentId(); - UUID b = factory.newBulkSegmentId(); - UUID c = factory.newDataSegmentId(); + SegmentId a = factory.newDataSegmentId(); + SegmentId b = factory.newBulkSegmentId(); + SegmentId c = factory.newDataSegmentId(); - Set<UUID> ids = factory.getReferencedSegmentIds(); + Set<SegmentId> ids = factory.getReferencedSegmentIds(); assertTrue(ids.contains(a)); assertTrue(ids.contains(b)); assertTrue(ids.contains(c)); @@ -68,19 +67,19 @@ public class SegmentIdFactoryTest { */ // @Test public void garbageCollection() { - UUID a = factory.newDataSegmentId(); - UUID b = factory.newBulkSegmentId(); + SegmentId a = factory.newDataSegmentId(); + SegmentId b = factory.newBulkSegmentId(); // generate lots of garbage copies of an UUID to get the // garbage collector to reclaim also the original instance for (int i = 0; i < 1000000; i++) { - a = new UUID( - a.getMostSignificantBits(), a.getLeastSignificantBits()); + a = new SegmentId( + null, a.getMostSignificantBits(), a.getLeastSignificantBits()); } System.gc(); // now the original UUID should no longer be present - Set<UUID> ids = factory.getReferencedSegmentIds(); + Set<SegmentId> ids = factory.getReferencedSegmentIds(); assertFalse(ids.contains(a)); assertTrue(ids.contains(b)); } Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (original) +++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java Thu Mar 13 14:31:57 2014 @@ -40,29 +40,29 @@ public class SegmentSizeTest { @Test public void testNodeSize() { NodeBuilder builder = EMPTY_NODE.builder(); - assertEquals(28, getSize(builder)); + assertEquals(40, getSize(builder)); assertEquals(4, getAmortizedSize(builder)); builder = EMPTY_NODE.builder(); builder.setProperty("foo", "bar"); - assertEquals(32, getSize(builder)); + assertEquals(44, getSize(builder)); assertEquals(8, getAmortizedSize(builder)); builder = EMPTY_NODE.builder(); builder.setProperty("foo", "bar"); builder.setProperty("baz", 123); - assertEquals(48, getSize(builder)); + assertEquals(60, getSize(builder)); assertEquals(12, getAmortizedSize(builder)); builder = EMPTY_NODE.builder(); builder.child("foo"); - assertEquals(48, getSize(builder)); + assertEquals(60, getSize(builder)); assertEquals(12, getAmortizedSize(builder)); builder = EMPTY_NODE.builder(); builder.child("foo"); builder.child("bar"); - assertEquals(76, getSize(builder)); + assertEquals(88, getSize(builder)); assertEquals(40, getAmortizedSize(builder)); } @@ -106,7 +106,7 @@ public class SegmentSizeTest { public void testAccessControlNodes() { NodeBuilder builder = EMPTY_NODE.builder(); builder.setProperty("jcr:primaryType", "rep:ACL", Type.NAME); - assertEquals(28, getSize(builder)); + assertEquals(40, getSize(builder)); assertEquals(4, getAmortizedSize(builder)); NodeBuilder deny = builder.child("deny"); @@ -114,7 +114,7 @@ public class SegmentSizeTest { deny.setProperty("rep:principalName", "everyone"); deny.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); - assertEquals(152, getSize(builder)); + assertEquals(164, getSize(builder)); assertEquals(28, getAmortizedSize(builder)); NodeBuilder allow = builder.child("allow"); @@ -122,7 +122,7 @@ public class SegmentSizeTest { allow.setProperty("rep:principalName", "administrators"); allow.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - assertEquals(272, getSize(builder)); + assertEquals(284, getSize(builder)); assertEquals(72, getAmortizedSize(builder)); NodeBuilder deny0 = builder.child("deny0"); @@ -131,7 +131,7 @@ public class SegmentSizeTest { deny0.setProperty("rep:glob", "*/activities/*"); builder.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES)); - assertEquals(364, getSize(builder)); + assertEquals(376, getSize(builder)); assertEquals(108, getAmortizedSize(builder)); NodeBuilder allow0 = builder.child("allow0"); @@ -139,7 +139,7 @@ public class SegmentSizeTest { allow0.setProperty("rep:principalName", "user-administrators"); allow0.setProperty(PropertyStates.createProperty( "rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES)); - assertEquals(420, getSize(builder)); + assertEquals(432, getSize(builder)); assertEquals(136, getAmortizedSize(builder)); } @@ -156,7 +156,7 @@ public class SegmentSizeTest { SegmentNodeState state = writer.writeNode(builder.getNodeState()); writer.flush(); Segment segment = store.readSegment(state.getRecordId().getSegmentId()); - assertEquals(27508, Segment.WEIGHER.weigh(null, segment)); + assertEquals(27520, segment.size()); writer.flush(); // force flushing of the previous segment @@ -165,7 +165,7 @@ public class SegmentSizeTest { state = writer.writeNode(builder.getNodeState()); writer.flush(); segment = store.readSegment(state.getRecordId().getSegmentId()); - assertEquals(484, Segment.WEIGHER.weigh(null, segment)); + assertEquals(496, segment.size()); } private int getSize(NodeBuilder builder) { @@ -174,7 +174,7 @@ public class SegmentSizeTest { RecordId id = writer.writeNode(builder.getNodeState()).getRecordId(); writer.flush(); Segment segment = store.readSegment(id.getSegmentId()); - return Segment.WEIGHER.weigh(null, segment); + return segment.size(); } private int getAmortizedSize(NodeBuilder builder) { @@ -184,7 +184,7 @@ public class SegmentSizeTest { RecordId id = writer.writeNode(state).getRecordId(); writer.flush(); Segment segment = store.readSegment(id.getSegmentId()); - int base = Segment.WEIGHER.weigh(null, segment); + int base = segment.size(); store = new MemoryStore(); // avoid cross-segment caching writer = store.getWriter(); @@ -192,7 +192,7 @@ public class SegmentSizeTest { id = writer.writeNode(state).getRecordId(); writer.flush(); segment = store.readSegment(id.getSegmentId()); - return Segment.WEIGHER.weigh(null, segment) - base - 4; + return segment.size() - base - 4; } } Modified: jackrabbit/oak/trunk/oak-http/src/main/java/org/apache/jackrabbit/oak/http/segment/SegmentServlet.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-http/src/main/java/org/apache/jackrabbit/oak/http/segment/SegmentServlet.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-http/src/main/java/org/apache/jackrabbit/oak/http/segment/SegmentServlet.java (original) +++ jackrabbit/oak/trunk/oak-http/src/main/java/org/apache/jackrabbit/oak/http/segment/SegmentServlet.java Thu Mar 13 14:31:57 2014 @@ -27,6 +27,7 @@ import javax.servlet.http.HttpServletRes import org.apache.jackrabbit.oak.plugins.segment.RecordId; import org.apache.jackrabbit.oak.plugins.segment.Segment; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState; import org.apache.jackrabbit.oak.plugins.segment.SegmentStore; @@ -36,9 +37,12 @@ public abstract class SegmentServlet ext protected abstract SegmentStore getSegmentStore(); - private UUID getSegmentId(String info) { + private SegmentId getSegmentId(String info) { try { - return UUID.fromString(info); + UUID uuid = UUID.fromString(info); + return getSegmentStore().getFactory().getSegmentId( + uuid.getMostSignificantBits(), + uuid.getLeastSignificantBits()); } catch (IllegalArgumentException e) { return null; } @@ -46,7 +50,8 @@ public abstract class SegmentServlet ext private RecordId getRecordId(BufferedReader reader) throws IOException { try { - return RecordId.fromString(reader.readLine()); + return RecordId.fromString( + getSegmentStore().getFactory(), reader.readLine()); } catch (IllegalArgumentException e) { return null; } @@ -79,8 +84,7 @@ public abstract class SegmentServlet ext response.sendError(HttpServletResponse.SC_BAD_REQUEST); } else { SegmentStore store = getSegmentStore(); - SegmentNodeState head = new SegmentNodeState( - store.getWriter().getDummySegment(), id); + SegmentNodeState head = new SegmentNodeState(id); if (store.setHead(store.getHead(), head)) { response.setStatus(HttpServletResponse.SC_OK); } else { @@ -97,13 +101,13 @@ public abstract class SegmentServlet ext private void doGetSegment( String info, HttpServletResponse response) throws ServletException, IOException { - UUID uuid = getSegmentId(info); - if (uuid == null) { + SegmentId id = getSegmentId(info); + if (id == null) { response.sendError(HttpServletResponse.SC_NOT_FOUND); return; } - Segment segment = getSegmentStore().readSegment(uuid); + Segment segment = id.getSegment(); if (segment == null) { response.sendError(HttpServletResponse.SC_NOT_FOUND); return; @@ -117,16 +121,16 @@ public abstract class SegmentServlet ext String info, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { - UUID uuid = getSegmentId(info); - if (uuid == null) { + SegmentId id = getSegmentId(info); + if (id == null) { response.sendError(HttpServletResponse.SC_NOT_FOUND); - } else if (getSegmentStore().readSegment(uuid) != null) { + } else if (getSegmentStore().containsSegment(id)) { // can't modify an existing segment response.sendError(HttpServletResponse.SC_FORBIDDEN); } else { // TODO: sanity check the segment data? byte[] data = ByteStreams.toByteArray(request.getInputStream()); - getSegmentStore().writeSegment(uuid, data, 0, data.length); + getSegmentStore().writeSegment(id, data, 0, data.length); response.setStatus(HttpServletResponse.SC_OK); } } Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1577175&r1=1577174&r2=1577175&view=diff ============================================================================== --- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original) +++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Thu Mar 13 14:31:57 2014 @@ -27,13 +27,16 @@ import java.util.Properties; import java.util.Queue; import java.util.Set; import java.util.UUID; + import javax.jcr.Repository; import com.google.common.collect.Maps; import com.google.common.collect.Queues; + import joptsimple.OptionParser; import joptsimple.OptionSet; import joptsimple.OptionSpec; + import org.apache.jackrabbit.core.RepositoryContext; import org.apache.jackrabbit.core.config.RepositoryConfig; import org.apache.jackrabbit.oak.Oak; @@ -44,7 +47,7 @@ import org.apache.jackrabbit.oak.http.Oa import org.apache.jackrabbit.oak.jcr.Jcr; import org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup; import org.apache.jackrabbit.oak.plugins.segment.Segment; -import org.apache.jackrabbit.oak.plugins.segment.SegmentIdFactory; +import org.apache.jackrabbit.oak.plugins.segment.SegmentId; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; @@ -153,22 +156,22 @@ public class Main { FileStore store = new FileStore(file, 256, false); try { if (args.length == 1) { - Map<UUID, List<UUID>> idmap = Maps.newHashMap(); + Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap(); int dataCount = 0; long dataSize = 0; int bulkCount = 0; long bulkSize = 0; - for (UUID uuid : store.getSegmentIds()) { - if (SegmentIdFactory.isDataSegmentId(uuid)) { - Segment segment = store.readSegment(uuid); + for (SegmentId id : store.getSegmentIds()) { + if (id.isDataSegmentId()) { + Segment segment = id.getSegment(); dataCount++; dataSize += segment.size(); - idmap.put(uuid, segment.getReferencedIds()); - } else if (SegmentIdFactory.isBulkSegmentId(uuid)) { + idmap.put(id, segment.getReferencedIds()); + } else if (id.isBulkSegmentId()) { bulkCount++; - bulkSize += store.readSegment(uuid).size(); - idmap.put(uuid, Collections.<UUID>emptyList()); + bulkSize += id.getSegment().size(); + idmap.put(id, Collections.<SegmentId>emptyList()); } } System.out.println("Total size:"); @@ -179,11 +182,11 @@ public class Main { "%6dMB in %6d bulk segments%n", bulkSize / (1024 * 1024), bulkCount); - Set<UUID> garbage = newHashSet(idmap.keySet()); - Queue<UUID> queue = Queues.newArrayDeque(); + Set<SegmentId> garbage = newHashSet(idmap.keySet()); + Queue<SegmentId> queue = Queues.newArrayDeque(); queue.add(store.getHead().getRecordId().getSegmentId()); while (!queue.isEmpty()) { - UUID id = queue.remove(); + SegmentId id = queue.remove(); if (garbage.remove(id)) { queue.addAll(idmap.get(id)); } @@ -192,13 +195,13 @@ public class Main { dataSize = 0; bulkCount = 0; bulkSize = 0; - for (UUID uuid : garbage) { - if (SegmentIdFactory.isDataSegmentId(uuid)) { + for (SegmentId id : garbage) { + if (id.isDataSegmentId()) { dataCount++; - dataSize += store.readSegment(uuid).size(); - } else if (SegmentIdFactory.isBulkSegmentId(uuid)) { + dataSize += id.getSegment().size(); + } else if (id.isBulkSegmentId()) { bulkCount++; - bulkSize += store.readSegment(uuid).size(); + bulkSize += id.getSegment().size(); } } System.out.println("Available for garbage collection:"); @@ -211,7 +214,10 @@ public class Main { } else { for (int i = 1; i < args.length; i++) { UUID uuid = UUID.fromString(args[i]); - System.out.println(store.readSegment(uuid)); + SegmentId id = store.getFactory().getSegmentId( + uuid.getMostSignificantBits(), + uuid.getLeastSignificantBits()); + System.out.println(id.getSegment()); } } } finally {
