Author: mduerig
Date: Thu Sep 15 12:59:23 2016
New Revision: 1760927
URL: http://svn.apache.org/viewvc?rev=1760927&view=rev
Log:
OAK-4740: TarReader recovery skips generating the index and binary graphs
Represent non inlined binary references by the record id of the string record
in the binary index
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java?rev=1760927&r1=1760926&r2=1760927&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
Thu Sep 15 12:59:23 2016
@@ -19,6 +19,12 @@ package org.apache.jackrabbit.oak.segmen
import java.util.UUID;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import com.google.common.base.Function;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+
/**
* Utility methods for working with binary references.
*/
@@ -42,4 +48,44 @@ public final class BinaryReferences {
};
}
+ /**
+ * @param blobId
+ * @return A new id for a binary reference from an inlined {@code blobId}.
+ */
+ @Nonnull
+ public static String newReference(@Nonnull String blobId) {
+ return "I" + blobId;
+ }
+
+ /**
+ * @param blobId
+ * @return A new id for a binary reference from a record id identifying a
+ * string record where the {@code blobId} is stored.
+ */
+ @Nonnull
+ public static String newReference(@Nonnull RecordId blobId) {
+ return "R" + blobId.toString10();
+ }
+
+ /**
+ * @param fileStore
+ * @return A new reader for binary references for the passed {@code
fileStore}.
+ * @see #newReference(String)
+ * @see #newReference(RecordId)
+ */
+ @Nonnull
+ public static Function<String, String> newReferenceReader(@Nonnull final
FileStore fileStore) {
+ return new Function<String, String>() {
+ @Nullable @Override
+ public String apply(String reference) {
+ if (reference.charAt(0) == 'I') {
+ return reference.substring(1);
+ } else {
+ return fileStore.getReader().readString(
+ RecordId.fromString(fileStore,
reference.substring(1)));
+ }
+ }
+
+ };
+ }
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1760927&r1=1760926&r2=1760927&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
Thu Sep 15 12:59:23 2016
@@ -44,6 +44,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.api.Type.NAME;
import static org.apache.jackrabbit.oak.api.Type.NAMES;
import static org.apache.jackrabbit.oak.api.Type.STRING;
+import static org.apache.jackrabbit.oak.segment.BinaryReferences.newReference;
import static org.apache.jackrabbit.oak.segment.MapRecord.BUCKETS_PER_LEVEL;
import static
org.apache.jackrabbit.oak.segment.RecordWriters.newNodeStateWriter;
@@ -798,13 +799,17 @@ public class SegmentWriter {
RecordId recordId;
+ String binaryReference;
if (data.length < Segment.BLOB_ID_SMALL_LIMIT) {
+ binaryReference = newReference(blobId);
recordId = RecordWriters.newBlobIdWriter(data).write(writer);
} else {
- recordId =
RecordWriters.newBlobIdWriter(writeString(blobId)).write(writer);
+ RecordId refId = writeString(blobId);
+ binaryReference = newReference(refId);
+ recordId = RecordWriters.newBlobIdWriter(refId).write(writer);
}
- binaryReferenceConsumer.consume(writer.getGeneration(),
recordId.asUUID(), blobId);
+ binaryReferenceConsumer.consume(writer.getGeneration(),
recordId.asUUID(), binaryReference);
return recordId;
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1760927&r1=1760926&r2=1760927&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Thu Sep 15 12:59:23 2016
@@ -37,6 +37,7 @@ import static java.util.concurrent.TimeU
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static
org.apache.jackrabbit.oak.segment.BinaryReferences.newReferenceReader;
import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
import static
org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder;
import static org.apache.jackrabbit.oak.segment.file.GCListener.Status.FAILURE;
@@ -925,7 +926,7 @@ public class FileStore implements Segmen
int minGeneration = getGcGeneration() -
gcOptions.getRetainedGenerations() + 1;
for (TarReader tarReader : tarReaders) {
- tarReader.collectBlobReferences(collector, minGeneration);
+ tarReader.collectBlobReferences(collector,
newReferenceReader(this), minGeneration);
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1760927&r1=1760926&r2=1760927&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
Thu Sep 15 12:59:23 2016
@@ -55,6 +55,7 @@ import java.util.zip.CRC32;
import javax.annotation.Nonnull;
import com.google.common.base.Charsets;
+import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
@@ -734,9 +735,13 @@ class TarReader implements Closeable {
* Collect the references of those blobs that are reachable from any
segment with a
* generation at or above {@code minGeneration}.
* @param collector
+ * @param referenceDecoder
* @param minGeneration
*/
- void collectBlobReferences(ReferenceCollector collector, int
minGeneration) {
+ void collectBlobReferences(
+ @Nonnull ReferenceCollector collector,
+ @Nonnull Function<String, String> referenceDecoder,
+ int minGeneration) {
Map<Integer, Map<UUID, Set<String>>> generations =
getBinaryReferences();
if (generations == null) {
@@ -750,7 +755,7 @@ class TarReader implements Closeable {
for (Set<String> references : entry.getValue().values()) {
for (String reference : references) {
- collector.addReference(reference, null);
+ collector.addReference(referenceDecoder.apply(reference),
null);
}
}
}
