Author: frm
Date: Tue Oct 18 11:10:44 2016
New Revision: 1765414

URL: http://svn.apache.org/viewvc?rev=1765414&view=rev
Log:
OAK-4947 - Inspect the segment on write to upadte the index of binary references

Removed:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferenceConsumer.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/BinaryReferences.java
Modified:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
 Tue Oct 18 11:10:44 2016
@@ -79,8 +79,7 @@ public class FileStoreBackupImpl impleme
                     backup.getReader(),
                     backup.getBlobStore(),
                     new WriterCacheManager.Default(),
-                    bufferWriter,
-                    backup.getBinaryReferenceConsumer()
+                    bufferWriter
             );
             Compactor compactor = new Compactor(
                     backup.getReader(),

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
 Tue Oct 18 11:10:44 2016
@@ -73,8 +73,7 @@ public class FileStoreRestoreImpl implem
                     store.getReader(),
                     store.getBlobStore(),
                     new WriterCacheManager.Default(),
-                    bufferWriter,
-                    store.getBinaryReferenceConsumer()
+                    bufferWriter
             );
             SegmentGCOptions gcOptions = defaultGCOptions().setOffline();
             Compactor compactor = new Compactor(

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
 Tue Oct 18 11:10:44 2016
@@ -157,7 +157,7 @@ public class SegmentBlob extends Record
     }
 
     @CheckForNull
-    static String readBlobId(@Nonnull Segment segment, int recordNumber) {
+    public static String readBlobId(@Nonnull Segment segment, int 
recordNumber) {
         byte head = segment.readByte(recordNumber);
         if ((head & 0xf0) == 0xe0) {
             // 1110 xxxx: external value, small blob ID

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
 Tue Oct 18 11:10:44 2016
@@ -44,7 +44,6 @@ import static org.apache.jackrabbit.oak.
 import static org.apache.jackrabbit.oak.api.Type.NAME;
 import static org.apache.jackrabbit.oak.api.Type.NAMES;
 import static org.apache.jackrabbit.oak.api.Type.STRING;
-import static org.apache.jackrabbit.oak.segment.BinaryReferences.newReference;
 import static org.apache.jackrabbit.oak.segment.MapRecord.BUCKETS_PER_LEVEL;
 import static 
org.apache.jackrabbit.oak.segment.RecordWriters.newNodeStateWriter;
 
@@ -115,9 +114,6 @@ public class SegmentWriter {
     private final WriteOperationHandler writeOperationHandler;
 
     @Nonnull
-    private final BinaryReferenceConsumer binaryReferenceConsumer;
-
-    @Nonnull
     private final SynchronizedDescriptiveStatistics nodeCompactTimeStats =
             new SynchronizedDescriptiveStatistics(NODE_WRITER_STATS_WINDOW);
 
@@ -135,19 +131,18 @@ public class SegmentWriter {
      * @param cacheManager  cache manager instance for the de-duplication 
caches used by this writer
      * @param writeOperationHandler  handler for write operations.
      */
-    public SegmentWriter(@Nonnull SegmentStore store,
-                         @Nonnull SegmentReader reader,
-                         @Nullable BlobStore blobStore,
-                         @Nonnull WriterCacheManager cacheManager,
-            @Nonnull WriteOperationHandler writeOperationHandler,
-            @Nonnull BinaryReferenceConsumer binaryReferenceConsumer
+    public SegmentWriter(
+            @Nonnull SegmentStore store,
+            @Nonnull SegmentReader reader,
+            @Nullable BlobStore blobStore,
+            @Nonnull WriterCacheManager cacheManager,
+            @Nonnull WriteOperationHandler writeOperationHandler
     ) {
         this.store = checkNotNull(store);
         this.reader = checkNotNull(reader);
         this.blobStore = blobStore;
         this.cacheManager = checkNotNull(cacheManager);
         this.writeOperationHandler = checkNotNull(writeOperationHandler);
-        this.binaryReferenceConsumer = checkNotNull(binaryReferenceConsumer);
     }
 
     /**
@@ -801,16 +796,12 @@ public class SegmentWriter {
 
             String binaryReference;
             if (data.length < Segment.BLOB_ID_SMALL_LIMIT) {
-                binaryReference = newReference(blobId);
                 recordId = RecordWriters.newBlobIdWriter(data).write(writer);
             } else {
                 RecordId refId = writeString(blobId);
-                binaryReference = newReference(refId);
                 recordId = RecordWriters.newBlobIdWriter(refId).write(writer);
             }
 
-            binaryReferenceConsumer.consume(writer.getGeneration(), 
recordId.asUUID(), binaryReference);
-
             return recordId;
         }
 

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriterBuilder.java
 Tue Oct 18 11:10:44 2016
@@ -148,8 +148,7 @@ public final class SegmentWriterBuilder
                 store.getReader(),
                 store.getBlobStore(),
                 cacheManager,
-                createWriter(store, pooled),
-                store.getBinaryReferenceConsumer()
+                createWriter(store, pooled)
         );
     }
 
@@ -163,8 +162,7 @@ public final class SegmentWriterBuilder
                 store.getReader(),
                 store.getBlobStore(),
                 cacheManager,
-                createWriter(store, pooled),
-                store.getBinaryReferenceConsumer()
+                createWriter(store, pooled)
         );
     }
 
@@ -178,8 +176,7 @@ public final class SegmentWriterBuilder
                 store.getReader(),
                 store.getBlobStore(),
                 cacheManager,
-                createWriter(store, pooled),
-                store.getBinaryReferenceConsumer()
+                createWriter(store, pooled)
         );
     }
 

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
 Tue Oct 18 11:10:44 2016
@@ -32,7 +32,6 @@ import static java.util.concurrent.TimeU
 import static java.util.concurrent.TimeUnit.SECONDS;
 import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
 import static 
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import static 
org.apache.jackrabbit.oak.segment.BinaryReferences.newReferenceReader;
 import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
 import static 
org.apache.jackrabbit.oak.segment.SegmentWriterBuilder.segmentWriterBuilder;
 import static org.apache.jackrabbit.oak.segment.file.GCListener.Status.FAILURE;
@@ -72,14 +71,15 @@ import com.google.common.base.Stopwatch;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
-
 import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics;
 import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
 import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
 import org.apache.jackrabbit.oak.segment.Compactor;
 import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.RecordType;
 import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.Segment.RecordConsumer;
+import org.apache.jackrabbit.oak.segment.SegmentBlob;
 import org.apache.jackrabbit.oak.segment.SegmentBufferWriter;
 import org.apache.jackrabbit.oak.segment.SegmentId;
 import org.apache.jackrabbit.oak.segment.SegmentIdTable;
@@ -113,9 +113,6 @@ public class FileStore extends AbstractF
     @Nonnull
     private final SegmentWriter segmentWriter;
 
-    @Nonnull
-    private final BinaryReferenceConsumer binaryReferenceConsumer;
-
     private final int maxFileSize;
 
     @Nonnull
@@ -169,18 +166,6 @@ public class FileStore extends AbstractF
                     + " is in use by another store.", ex);
         }
 
-        this.binaryReferenceConsumer = new BinaryReferenceConsumer() {
-            @Override
-            public void consume(int generation, UUID segmentId, String 
binaryReference) {
-                fileStoreLock.writeLock().lock();
-                try {
-                    tarWriter.addBinaryReference(generation, segmentId, 
binaryReference);
-                } finally {
-                    fileStoreLock.writeLock().unlock();
-                }
-            }
-        };
-
         this.segmentWriter = segmentWriterBuilder("sys")
                 .withGeneration(new Supplier<Integer>() {
                     @Override
@@ -458,11 +443,6 @@ public class FileStore extends AbstractF
         return segmentWriter;
     }
 
-    @Nonnull
-    public BinaryReferenceConsumer getBinaryReferenceConsumer() {
-        return binaryReferenceConsumer;
-    }
-
     @Override
     @Nonnull
     public TarRevisions getRevisions() {
@@ -661,11 +641,8 @@ public class FileStore extends AbstractF
             // (potentially) flushing the TAR file.
 
             if (segment != null) {
-                UUID from = segment.getSegmentId().asUUID();
-
-                for (int i = 0; i < segment.getReferencedSegmentIdCount(); 
i++) {
-                    tarWriter.addGraphEdge(from, 
segment.getReferencedSegmentId(i));
-                }
+                populateTarGraph(segment);
+                populateTarBinaryReferences(segment);
             }
 
             // Close the TAR file if the size exceeds the maximum.
@@ -684,6 +661,28 @@ public class FileStore extends AbstractF
         }
     }
 
+    private void populateTarGraph(Segment segment) {
+        UUID from = segment.getSegmentId().asUUID();
+        for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
+            tarWriter.addGraphEdge(from, segment.getReferencedSegmentId(i));
+        }
+    }
+
+    private void populateTarBinaryReferences(final Segment segment) {
+        final int generation = segment.getGcGeneration();
+        final UUID id = segment.getSegmentId().asUUID();
+        segment.forEachRecord(new RecordConsumer() {
+
+            @Override
+            public void consume(int number, RecordType type, int offset) {
+                if (type == RecordType.BLOB_ID) {
+                    tarWriter.addBinaryReference(generation, id, 
SegmentBlob.readBlobId(segment, number));
+                }
+            }
+
+        });
+    }
+
     /**
      * Switch to a new tar writer.
      * This method may only be called when holding the write lock of {@link 
#fileStoreLock}
@@ -976,7 +975,7 @@ public class FileStore extends AbstractF
         throws IOException {
             if (gcOptions.isOffline()) {
                 BlobStore blobStore = getBlobStore();
-                SegmentWriter writer = new SegmentWriter(FileStore.this, 
segmentReader, blobStore, new Default(), bufferWriter, binaryReferenceConsumer);
+                SegmentWriter writer = new SegmentWriter(FileStore.this, 
segmentReader, blobStore, new Default(), bufferWriter);
                 return new Compactor(segmentReader, writer, blobStore, cancel, 
gcOptions)
                         .compact(EMPTY_NODE, head, EMPTY_NODE);
             } else {
@@ -1209,7 +1208,7 @@ public class FileStore extends AbstractF
 
             int minGeneration = getGcGeneration() - 
gcOptions.getRetainedGenerations() + 1;
             for (TarReader tarReader : tarReaders) {
-                tarReader.collectBlobReferences(collector, 
newReferenceReader(FileStore.this), minGeneration);
+                tarReader.collectBlobReferences(collector, minGeneration);
             }
         }
 

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
 Tue Oct 18 11:10:44 2016
@@ -55,7 +55,6 @@ import java.util.zip.CRC32;
 import javax.annotation.Nonnull;
 
 import com.google.common.base.Charsets;
-import com.google.common.base.Function;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Sets;
 import org.apache.commons.io.FileUtils;
@@ -735,13 +734,9 @@ class TarReader implements Closeable {
      * Collect the references of those blobs that are reachable from any 
segment with a
      * generation at or above {@code minGeneration}.
      * @param collector
-     * @param referenceDecoder
      * @param minGeneration
      */
-    void collectBlobReferences(
-            @Nonnull ReferenceCollector collector,
-            @Nonnull Function<String, String> referenceDecoder,
-            int minGeneration) {
+    void collectBlobReferences(@Nonnull ReferenceCollector collector, int 
minGeneration) {
         Map<Integer, Map<UUID, Set<String>>> generations = 
getBinaryReferences();
 
         if (generations == null) {
@@ -755,7 +750,7 @@ class TarReader implements Closeable {
 
             for (Set<String> references : entry.getValue().values()) {
                 for (String reference : references) {
-                    collector.addReference(referenceDecoder.apply(reference), 
null);
+                    collector.addReference(reference, null);
                 }
             }
         }

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/http/HttpStore.java
 Tue Oct 18 11:10:44 2016
@@ -35,8 +35,6 @@ import javax.annotation.Nonnull;
 
 import com.google.common.base.Supplier;
 import com.google.common.io.ByteStreams;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
-import org.apache.jackrabbit.oak.segment.BinaryReferences;
 import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
 import org.apache.jackrabbit.oak.segment.Revisions;
 import org.apache.jackrabbit.oak.segment.Segment;
@@ -113,11 +111,6 @@ public class HttpStore implements Segmen
         return revisions;
     }
 
-    @Nonnull
-    public BinaryReferenceConsumer getBinaryReferenceConsumer() {
-        return BinaryReferences.newDiscardBinaryReferenceConsumer();
-    }
-
     @Override
     @Nonnull
     public SegmentId newSegmentId(long msb, long lsb) {

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java?rev=1765414&r1=1765413&r2=1765414&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/memory/MemoryStore.java
 Tue Oct 18 11:10:44 2016
@@ -29,8 +29,6 @@ import javax.annotation.Nonnull;
 
 import com.google.common.base.Supplier;
 import com.google.common.collect.Maps;
-import org.apache.jackrabbit.oak.segment.BinaryReferenceConsumer;
-import org.apache.jackrabbit.oak.segment.BinaryReferences;
 import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
 import org.apache.jackrabbit.oak.segment.Revisions;
 import org.apache.jackrabbit.oak.segment.Segment;
@@ -107,11 +105,6 @@ public class MemoryStore implements Segm
         return revisions;
     }
 
-    @Nonnull
-    public BinaryReferenceConsumer getBinaryReferenceConsumer() {
-        return BinaryReferences.newDiscardBinaryReferenceConsumer();
-    }
-
     @Override
     public boolean containsSegment(SegmentId id) {
         return id.sameStore(this) || segments.containsKey(id);


Reply via email to