Author: mduerig
Date: Wed Apr 20 10:03:45 2016
New Revision: 1740087
URL: http://svn.apache.org/viewvc?rev=1740087&view=rev
Log:
OAK-3348: Cross gc sessions might introduce references to pre-compacted segments
* Move record caches to top level
* Re-enable template and string caches and including gc generation in the cache
key, thus there is no need anymore to flush the caches on compaction.
Added:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordCache.java
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Added:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordCache.java?rev=1740087&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordCache.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/RecordCache.java
Wed Apr 20 10:03:45 2016
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * FIXME OAK-3348 XXX document
+ */
+abstract class RecordCache<T> {
+ // FIXME OAK-3348 XXX this caches retain in mem refs to old gens. assess
impact and mitigate/fix
+ private static final RecordCache<?> DISABLED_CACHE = new
RecordCache<Object>() {
+ @Override
+ RecordId get(Object key) {
+ return null;
+ }
+
+ @Override
+ RecordId put(Object key, RecordId value) {
+ return null;
+ }
+
+ @Override
+ void clear() { }
+ };
+
+ @SuppressWarnings("unchecked")
+ static <T> RecordCache<T> newRecordCache(final int initialSize, final int
size, boolean disabled) {
+ if (disabled) {
+ return (RecordCache<T>) DISABLED_CACHE;
+ } else {
+ return new LRURecordCache<T>(initialSize, size);
+ }
+ }
+
+ abstract RecordId get(Object key);
+ abstract RecordId put(T key, RecordId value);
+ abstract void clear();
+
+ private static final class LRURecordCache<T> extends RecordCache<T> {
+ private final Map<T, RecordId> cache;
+
+ private LRURecordCache(int initialSize, final int size) {
+ cache = new LinkedHashMap<T, RecordId>(initialSize, 0.9f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<T, RecordId>
eldest) {
+ return size() > size;
+ }
+ };
+ }
+
+ @Override
+ synchronized RecordId get(Object key) {
+ return cache.get(key);
+ }
+
+ @Override
+ synchronized RecordId put(T key, RecordId value) {
+ return cache.put(key, value);
+ }
+
+ @Override
+ public synchronized void clear() {
+ cache.clear();
+ }
+ }
+}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1740087&r1=1740086&r2=1740087&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
Wed Apr 20 10:03:45 2016
@@ -40,6 +40,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.NAMES;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static
org.apache.jackrabbit.oak.plugins.segment.MapRecord.BUCKETS_PER_LEVEL;
+import static
org.apache.jackrabbit.oak.plugins.segment.RecordCache.newRecordCache;
import static
org.apache.jackrabbit.oak.plugins.segment.RecordWriters.newBlobIdWriter;
import static
org.apache.jackrabbit.oak.plugins.segment.RecordWriters.newBlockWriter;
import static
org.apache.jackrabbit.oak.plugins.segment.RecordWriters.newListBucketWriter;
@@ -60,12 +61,12 @@ import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Arrays;
import java.util.Collection;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.jcr.PropertyType;
+import com.google.common.base.Objects;
import com.google.common.io.Closeables;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -94,8 +95,9 @@ public class SegmentWriter {
* Cache of recently stored string records, used to avoid storing
duplicates
* of frequently occurring data.
*/
- private final Map<String, RecordId> stringCache = newItemsCache(
- STRING_RECORDS_CACHE_SIZE);
+ private final RecordCache<Object> stringCache = newRecordCache(
+ STRING_RECORDS_CACHE_SIZE <= 0 ? 0 : (int) (STRING_RECORDS_CACHE_SIZE
* 1.2),
+ STRING_RECORDS_CACHE_SIZE, STRING_RECORDS_CACHE_SIZE <= 0);
private static final int TPL_RECORDS_CACHE_SIZE = Integer.getInteger(
"oak.segment.writer.templatesCacheSize", 3000);
@@ -104,7 +106,9 @@ public class SegmentWriter {
* Cache of recently stored template records, used to avoid storing
* duplicates of frequently occurring data.
*/
- private final Map<Template, RecordId> templateCache =
newItemsCache(TPL_RECORDS_CACHE_SIZE);
+ private final RecordCache<Object> templateCache = newRecordCache(
+ TPL_RECORDS_CACHE_SIZE <= 0 ? 0 : (int) (TPL_RECORDS_CACHE_SIZE * 1.2),
+ TPL_RECORDS_CACHE_SIZE, TPL_RECORDS_CACHE_SIZE <= 0);
private final SegmentStore store;
@@ -115,35 +119,6 @@ public class SegmentWriter {
private final SegmentBufferWriterPool segmentBufferWriterPool;
- private static final <T> Map<T, RecordId> newItemsCache(final int size) {
- final boolean disabled = true; // FIXME OAK-3348 re-enable caches but
make generation part of cache key to avoid backrefs
- final int safeSize = size <= 0 ? 0 : (int) (size * 1.2);
- return new LinkedHashMap<T, RecordId>(safeSize, 0.9f, true) {
- @Override
- protected boolean removeEldestEntry(Map.Entry<T, RecordId> e) {
- return size() > size;
- }
- @Override
- public synchronized RecordId get(Object key) {
- if (disabled) {
- return null;
- }
- return super.get(key);
- }
- @Override
- public synchronized RecordId put(T key, RecordId value) {
- if (disabled) {
- return null;
- }
- return super.put(key, value);
- }
- @Override
- public synchronized void clear() {
- super.clear();
- }
- };
- }
-
/**
* @param store store to write to
* @param version segment version to write
@@ -159,12 +134,6 @@ public class SegmentWriter {
segmentBufferWriterPool.flush();
}
- // FIXME OAK-3348 this is a hack and probably prone to races: replace with
making the tacker allocate a new writer
- public void dropCache() {
- stringCache.clear();
- templateCache.clear();
- }
-
MapRecord writeMap(MapRecord base, Map<String, RecordId> changes) throws
IOException {
Writer writer = new Writer();
try {
@@ -497,7 +466,7 @@ public class SegmentWriter {
* @return value record identifier
*/
private RecordId writeString(String string) throws IOException {
- RecordId id = stringCache.get(string);
+ RecordId id = stringCache.get(key(string));
if (id != null) {
return id; // shortcut if the same string was recently stored
}
@@ -507,7 +476,7 @@ public class SegmentWriter {
if (data.length < Segment.MEDIUM_LIMIT) {
// only cache short strings to avoid excessive memory use
id = writeValueRecord(data.length, data);
- stringCache.put(string, id);
+ stringCache.put(key(string), id);
return id;
}
@@ -691,7 +660,7 @@ public class SegmentWriter {
private RecordId writeTemplate(Template template) throws IOException {
checkNotNull(template);
- RecordId id = templateCache.get(template);
+ RecordId id = templateCache.get(key(template));
if (id != null) {
return id; // shortcut if the same template was recently stored
}
@@ -762,7 +731,7 @@ public class SegmentWriter {
RecordId tid = newTemplateWriter(ids, propertyNames,
propertyTypes, head, primaryId, mixinIds, childNameId,
propNamesId, version).write(writer);
- templateCache.put(template, tid);
+ templateCache.put(key(template), tid);
return tid;
}
@@ -904,6 +873,10 @@ public class SegmentWriter {
return thatGen < thisGen;
}
+ private <T> Key<T> key(T t) {
+ return new Key<T>(t, writer.getGeneration());
+ }
+
private class ChildNodeCollectorDiff extends DefaultNodeStateDiff {
private final Map<String, RecordId> childNodes = newHashMap();
private IOException exception;
@@ -952,4 +925,32 @@ public class SegmentWriter {
return store.getTracker();
}
+ private static final class Key<T> {
+ private final T t;
+ private final int generation;
+
+ private Key(T t, int generation) {
+ this.t = t;
+ this.generation = generation;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ Key<?> that = (Key<?>) other;
+ return generation == that.generation && t.equals(that.t);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(t, generation);
+ }
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1740087&r1=1740086&r2=1740087&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-next/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Wed Apr 20 10:03:45 2016
@@ -1136,7 +1136,7 @@ public class FileStore implements Segmen
closeAndLogOnFail(diskSpaceThread);
try {
flush();
- tracker.getWriter().dropCache();
+ // FIXME OAK-3348 XXX replace with some sort of close call
tracker.getWriter().dropCache();
fileStoreLock.writeLock().lock();
try {
closeAndLogOnFail(writer);
@@ -1476,11 +1476,6 @@ public class FileStore implements Segmen
if (setHead(before, after)) {
tracker.setCompactionMap(compactor.getCompactionMap());
- // Drop the SegmentWriter caches and flush any existing state
- // in an attempt to prevent new references to old pre-compacted
- // content. TODO: There should be a cleaner way to do this.
(implement GCMonitor!?)
- tracker.getWriter().dropCache();
-
CompactionMap cm = tracker.getCompactionMap();
gcMonitor.compacted(cm.getSegmentCounts(),
cm.getRecordCounts(), cm.getEstimatedWeights());
tracker.clearSegmentIdTables(compactionStrategy);
