Author: frm
Date: Tue Aug  8 12:50:42 2017
New Revision: 1804413

URL: http://svn.apache.org/viewvc?rev=1804413&view=rev
Log:
OAK-6528 - Transparently read the two different formats of binary references 
index

Added:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/ReaderAtEnd.java
      - copied, changed from r1804412, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/ReaderAtEnd.java
Removed:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/ReaderAtEnd.java
Modified:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV1.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
 Tue Aug  8 12:50:42 2017
@@ -45,11 +45,6 @@ class TarConstants {
     static final int GRAPH_MAGIC = ('\n' << 24) + ('0' << 16) + ('G' << 8) + 
'\n';
 
     /**
-     * Magic sequence at the end of the binary references block.
-     */
-    static final int BINARY_REFERENCES_MAGIC = ('\n' << 24) + ('0' << 16) + 
('B' << 8) + '\n';
-
-    /**
      * The tar file block size.
      */
     static final int BLOCK_SIZE = 512;

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
 Tue Aug  8 12:50:42 2017
@@ -29,7 +29,6 @@ import static com.google.common.collect.
 import static java.nio.ByteBuffer.wrap;
 import static java.util.Collections.singletonList;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
-import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BINARY_REFERENCES_MAGIC;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.index.IndexLoader.newIndexLoader;
@@ -56,16 +55,17 @@ import java.util.zip.CRC32;
 
 import javax.annotation.Nonnull;
 
-import com.google.common.base.Charsets;
 import com.google.common.base.Predicate;
 import com.google.common.base.Stopwatch;
-import com.google.common.collect.Sets;
 import org.apache.commons.io.FileUtils;
+import 
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndex;
+import 
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexLoader;
+import 
org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException;
 import org.apache.jackrabbit.oak.segment.file.tar.index.Index;
 import org.apache.jackrabbit.oak.segment.file.tar.index.IndexEntry;
 import org.apache.jackrabbit.oak.segment.file.tar.index.IndexLoader;
 import org.apache.jackrabbit.oak.segment.file.tar.index.InvalidIndexException;
-import org.apache.jackrabbit.oak.segment.file.tar.index.ReaderAtEnd;
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -599,23 +599,18 @@ class TarReader implements Closeable {
      * @param skipGeneration An instance of {@link Predicate}.
      */
     void collectBlobReferences(@Nonnull Consumer<String> collector, 
Predicate<GCGeneration> skipGeneration) {
-        Map<GCGeneration, Map<UUID, Set<String>>> generations = 
getBinaryReferences();
+        BinaryReferencesIndex references = getBinaryReferences();
 
-        if (generations == null) {
+        if (references == null) {
             return;
         }
 
-        for (Entry<GCGeneration, Map<UUID, Set<String>>> entry : 
generations.entrySet()) {
-            if (skipGeneration.apply(entry.getKey())) {
-                continue;
-            }
-
-            for (Set<String> references : entry.getValue().values()) {
-                for (String reference : references) {
-                    collector.accept(reference);
-                }
+        references.forEach((generation, full, compacted, segment, reference) 
-> {
+            if (skipGeneration.apply(newGCGeneration(generation, full, 
compacted))) {
+                return;
             }
-        }
+            collector.accept(reference);
+        });
     }
 
     /**
@@ -801,18 +796,15 @@ class TarReader implements Closeable {
 
         // Reconstruct the binary reference index for non-cleaned segments.
 
-        Map<GCGeneration, Map<UUID, Set<String>>> references = 
getBinaryReferences();
+        BinaryReferencesIndex references = getBinaryReferences();
 
-        for (Entry<GCGeneration, Map<UUID, Set<String>>> ge : 
references.entrySet()) {
-            for (Entry<UUID, Set<String>> se : ge.getValue().entrySet()) {
-                if (cleaned.contains(se.getKey())) {
-                    continue;
-                }
-                for (String reference : se.getValue()) {
-                    writer.addBinaryReference(ge.getKey(), se.getKey(), 
reference);
+        if (references != null) {
+            references.forEach((gen, full, compacted, id, reference) -> {
+                if (cleaned.contains(id)) {
+                    return;
                 }
-            }
-
+                writer.addBinaryReference(newGCGeneration(gen, full, 
compacted), id, reference);
+            });
         }
 
         writer.close();
@@ -888,97 +880,19 @@ class TarReader implements Closeable {
      *
      * @return An instance of {@link Map}.
      */
-    Map<GCGeneration, Map<UUID, Set<String>>> getBinaryReferences() {
-        ByteBuffer buffer;
-
+    BinaryReferencesIndex getBinaryReferences() {
+        BinaryReferencesIndex index = null;
         try {
-            buffer = loadBinaryReferences();
-        } catch (IOException e) {
+            index = loadBinaryReferences();
+        } catch (InvalidBinaryReferencesIndexException | IOException e) {
             log.warn("Exception while loading binary reference", e);
-            return null;
         }
-
-        if (buffer == null) {
-            return null;
-        }
-
-        return parseBinaryReferences(buffer);
+        return index;
     }
 
-    private ByteBuffer loadBinaryReferences() throws IOException {
+    private BinaryReferencesIndex loadBinaryReferences() throws IOException, 
InvalidBinaryReferencesIndexException {
         int end = access.length() - 2 * BLOCK_SIZE - getIndexEntrySize() - 
getGraphEntrySize();
-
-        ByteBuffer meta = access.read(end - 16, 16);
-
-        int crc32 = meta.getInt();
-        int count = meta.getInt();
-        int size = meta.getInt();
-        int magic = meta.getInt();
-
-        if (magic != BINARY_REFERENCES_MAGIC) {
-            log.warn("Invalid binary references magic number");
-            return null;
-        }
-
-        if (count < 0 || size < count * 22 + 16) {
-            log.warn("Invalid binary references size or count");
-            return null;
-        }
-
-        ByteBuffer buffer = access.read(end - size, size);
-
-        byte[] data = new byte[size - 16];
-        buffer.mark();
-        buffer.get(data);
-        buffer.reset();
-
-        CRC32 checksum = new CRC32();
-        checksum.update(data);
-
-        if ((int) (checksum.getValue()) != crc32) {
-            log.warn("Invalid binary references checksum");
-            return null;
-        }
-
-        return buffer;
-    }
-
-    private static Map<GCGeneration, Map<UUID, Set<String>>> 
parseBinaryReferences(ByteBuffer buffer) {
-        int nGenerations = buffer.getInt(buffer.limit() - 12);
-
-        Map<GCGeneration, Map<UUID, Set<String>>> binaryReferences = 
newHashMapWithExpectedSize(nGenerations);
-
-        for (int i = 0; i < nGenerations; i++) {
-            int generation = buffer.getInt();
-            int fullGeneration = buffer.getInt();
-            boolean isCompacted = buffer.get() != 0;
-            int segmentCount = buffer.getInt();
-
-            Map<UUID, Set<String>> segments = 
newHashMapWithExpectedSize(segmentCount);
-
-            for (int j = 0; j < segmentCount; j++) {
-                long msb = buffer.getLong();
-                long lsb = buffer.getLong();
-                int referenceCount = buffer.getInt();
-
-                Set<String> references = 
Sets.newHashSetWithExpectedSize(referenceCount);
-
-                for (int k = 0; k < referenceCount; k++) {
-                    int length = buffer.getInt();
-
-                    byte[] data = new byte[length];
-                    buffer.get(data);
-
-                    references.add(new String(data, Charsets.UTF_8));
-                }
-
-                segments.put(new UUID(msb, lsb), references);
-            }
-
-            binaryReferences.put(newGCGeneration(generation, fullGeneration, 
isCompacted), segments);
-        }
-
-        return binaryReferences;
+        return BinaryReferencesIndexLoader.loadBinaryReferencesIndex((whence, 
size) -> access.read(end - whence, size));
     }
 
     /**

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
 Tue Aug  8 12:50:42 2017
@@ -27,10 +27,10 @@ import static com.google.common.collect.
 import static com.google.common.collect.Maps.newLinkedHashMap;
 import static com.google.common.collect.Sets.newHashSet;
 import static java.lang.String.format;
-import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BINARY_REFERENCES_MAGIC;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.FILE_NAME_FORMAT;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC;
+import static 
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexWriter.newBinaryReferencesIndexWriter;
 
 import java.io.Closeable;
 import java.io.File;
@@ -46,8 +46,8 @@ import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.zip.CRC32;
 
-import com.google.common.base.Charsets;
 import com.google.common.base.Stopwatch;
+import 
org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexWriter;
 import org.apache.jackrabbit.oak.segment.file.tar.index.IndexWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -114,7 +114,7 @@ class TarWriter implements Closeable {
     /**
      * List of binary references contained in this TAR file.
      */
-    private final Map<GCGeneration, Map<UUID, Set<String>>> binaryReferences = 
newHashMap();
+    private final BinaryReferencesIndexWriter binaryReferences = 
newBinaryReferencesIndexWriter();
 
     /**
      * Graph of references between segments.
@@ -223,10 +223,13 @@ class TarWriter implements Closeable {
     }
 
     void addBinaryReference(GCGeneration generation, UUID segmentId, String 
reference) {
-        binaryReferences
-                .computeIfAbsent(generation, k -> newHashMap())
-                .computeIfAbsent(segmentId, k -> newHashSet())
-                .add(reference);
+        binaryReferences.addEntry(
+            generation.getGeneration(),
+            generation.getFullGeneration(),
+            generation.isCompacted(),
+            segmentId,
+            reference
+        );
     }
 
     void addGraphEdge(UUID from, UUID to) {
@@ -318,105 +321,14 @@ class TarWriter implements Closeable {
     }
 
     private void writeBinaryReferences() throws IOException {
-        int binaryReferenceSize = 0;
-
-        // The following information are stored in the footer as meta-
-        // information about the entry.
-
-        // 4 bytes to store a magic number identifying this entry as containing
-        // references to binary values.
-        binaryReferenceSize += 4;
-
-        // 4 bytes to store the CRC32 checksum of the data in this entry.
-        binaryReferenceSize += 4;
-
-        // 4 bytes to store the length of this entry, without including the
-        // optional padding.
-        binaryReferenceSize += 4;
-
-        // 4 bytes to store the number of generations pairs in the binary
-        // references map.
-        binaryReferenceSize += 4;
-
-        // The following information are stored as part of the main content of
-        // this entry, after the optional padding.
-
-        for (Map<UUID, Set<String>> segmentToReferences : 
binaryReferences.values()) {
-            // 4 bytes per generation to store the full generation number.
-            binaryReferenceSize += 4;
-
-            // 4 bytes per generation to store the tail generation number.
-            binaryReferenceSize += 4;
-
-            // 1 byte per generation to store the "tail" flag.
-            binaryReferenceSize += 1;
-
-            // 4 bytes per generation to store the number of segments.
-            binaryReferenceSize += 4;
-
-            for (Set<String> references : segmentToReferences.values()) {
-                // 16 bytes per segment identifier.
-                binaryReferenceSize += 16;
-
-                // 4 bytes to store the number of references for this segment.
-                binaryReferenceSize += 4;
-
-                for (String reference : references) {
-                    // 4 bytes for each reference to store the length of the 
reference.
-                    binaryReferenceSize += 4;
-
-                    // A variable amount of bytes, depending on the reference 
itself.
-                    binaryReferenceSize += 
reference.getBytes(Charsets.UTF_8).length;
-                }
-            }
-        }
-
-        ByteBuffer buffer = ByteBuffer.allocate(binaryReferenceSize);
-
-        for (Entry<GCGeneration, Map<UUID, Set<String>>> be : 
binaryReferences.entrySet()) {
-            GCGeneration generation = be.getKey();
-            Map<UUID, Set<String>> segmentToReferences = be.getValue();
-
-            buffer.putInt(generation.getGeneration());
-            buffer.putInt(generation.getFullGeneration());
-            buffer.put((byte) (generation.isCompacted() ? 1 : 0));
-            buffer.putInt(segmentToReferences.size());
-
-            for (Entry<UUID, Set<String>> se : segmentToReferences.entrySet()) 
{
-                UUID segmentId = se.getKey();
-                Set<String> references = se.getValue();
-
-                buffer.putLong(segmentId.getMostSignificantBits());
-                buffer.putLong(segmentId.getLeastSignificantBits());
-                buffer.putInt(references.size());
-
-                for (String reference : references) {
-                    byte[] bytes = reference.getBytes(Charsets.UTF_8);
-
-                    buffer.putInt(bytes.length);
-                    buffer.put(bytes);
-                }
-            }
-        }
-
-        CRC32 checksum = new CRC32();
-        checksum.update(buffer.array(), 0, buffer.position());
-        buffer.putInt((int) checksum.getValue());
-        buffer.putInt(binaryReferences.size());
-        buffer.putInt(binaryReferenceSize);
-        buffer.putInt(BINARY_REFERENCES_MAGIC);
-
-        int paddingSize = getPaddingSize(binaryReferenceSize);
-
-        byte[] header = newEntryHeader(file.getName() + ".brf", 
binaryReferenceSize + paddingSize);
-
+        byte[] data = binaryReferences.write();
+        int paddingSize = getPaddingSize(data.length);
+        byte[] header = newEntryHeader(file.getName() + ".brf", data.length + 
paddingSize);
         access.write(header);
-
         if (paddingSize > 0) {
             access.write(ZERO_BYTES, 0, paddingSize);
         }
-
-        access.write(buffer.array());
+        access.write(data);
     }
 
     private void writeGraph() throws IOException {

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+/**
+ * An index of binary references.
+ */
+public class BinaryReferencesIndex {
+
+    /**
+     * A consumer of entries from a binary references index.
+     */
+    public interface EntryConsumer {
+
+        /**
+         * Consume an entry from a binary references index.
+         *
+         * @param generation The generation of the segment containing the 
binary
+         *                   reference.
+         * @param full       The full generation of the segment containing the
+         *                   binary reference.
+         * @param compacted  {@code true} if the segment was created by a
+         *                   compaction operation.
+         * @param segment    The identifier of the segment containing the 
binary
+         *                   reference.
+         * @param reference  The binary reference.
+         */
+        void consume(int generation, int full, boolean compacted, UUID 
segment, String reference);
+
+    }
+
+    private final Map<Generation, Map<UUID, Set<String>>> references;
+
+    BinaryReferencesIndex(Map<Generation, Map<UUID, Set<String>>> references) {
+        this.references = references;
+    }
+
+    /**
+     * Iterate over every entry in this index.
+     *
+     * @param consumer An instance of {@link EntryConsumer}.
+     */
+    public void forEach(EntryConsumer consumer) {
+        references.forEach((generation, entries) -> {
+            entries.forEach((segment, references) -> {
+                references.forEach(reference -> {
+                    consumer.consume(
+                        generation.generation,
+                        generation.full,
+                        generation.compacted,
+                        segment,
+                        reference
+                    );
+                });
+            });
+        });
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndex.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
+public class BinaryReferencesIndexLoader {
+
+    private BinaryReferencesIndexLoader() {
+        // Prevent external instantiation
+    }
+
+    /**
+     * Load and validate a binary references index. The binary references index
+     * is read backward. The medium the index is read from is represented by an
+     * instance of {@link ReaderAtEnd}. The {@link ReaderAtEnd} should behave 
as
+     * it is positioned at the end of the binary references index.
+     *
+     * @param reader An instance of {@link ReaderAtEnd}.
+     * @return The binary references index.
+     * @throws IOException                           if an error occurs while
+     *                                               reading the index.
+     * @throws InvalidBinaryReferencesIndexException if the index is invalid or
+     *                                               malformed.
+     */
+    public static BinaryReferencesIndex loadBinaryReferencesIndex(ReaderAtEnd 
reader) throws IOException, InvalidBinaryReferencesIndexException {
+        switch (readMagic(reader)) {
+            case BinaryReferencesIndexLoaderV1.MAGIC:
+                return 
BinaryReferencesIndexLoaderV1.loadBinaryReferencesIndex(reader);
+            case BinaryReferencesIndexLoaderV2.MAGIC:
+                return 
BinaryReferencesIndexLoaderV2.loadBinaryReferencesIndex(reader);
+            default:
+                throw new InvalidBinaryReferencesIndexException("Unrecognized 
magic number");
+        }
+    }
+
+    private static int readMagic(ReaderAtEnd reader) throws IOException {
+        return reader.readAtEnd(Integer.BYTES, Integer.BYTES).getInt();
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoader.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.CRC32;
+
+import com.google.common.base.Charsets;
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
+class BinaryReferencesIndexLoaderV1 {
+
+    static final int MAGIC = ('\n' << 24) + ('0' << 16) + ('B' << 8) + '\n';
+
+    private static final int FOOTER_SIZE = 16;
+
+    static BinaryReferencesIndex loadBinaryReferencesIndex(ReaderAtEnd reader) 
throws IOException, InvalidBinaryReferencesIndexException {
+        ByteBuffer meta = reader.readAtEnd(FOOTER_SIZE, FOOTER_SIZE);
+
+        int crc32 = meta.getInt();
+        int count = meta.getInt();
+        int size = meta.getInt();
+        int magic = meta.getInt();
+
+        if (magic != MAGIC) {
+            throw new InvalidBinaryReferencesIndexException("Invalid magic 
number");
+        }
+        if (count < 0) {
+            throw new InvalidBinaryReferencesIndexException("Invalid count");
+        }
+        if (size < count * 22 + 16) {
+            throw new InvalidBinaryReferencesIndexException("Invalid size");
+        }
+
+        ByteBuffer buffer = reader.readAtEnd(size, size - FOOTER_SIZE);
+
+        CRC32 checksum = new CRC32();
+        byte[] data = new byte[size - FOOTER_SIZE];
+        buffer.mark();
+        buffer.get(data);
+        buffer.reset();
+        checksum.update(data);
+
+        if ((int) (checksum.getValue()) != crc32) {
+            throw new InvalidBinaryReferencesIndexException("Invalid 
checksum");
+        }
+
+        return new BinaryReferencesIndex(parseBinaryReferencesIndex(count, 
buffer));
+    }
+
+    private static Map<Generation, Map<UUID, Set<String>>> 
parseBinaryReferencesIndex(int count, ByteBuffer buffer) {
+        Map<Generation, Map<UUID, Set<String>>> result = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            Generation k = parseGeneration(buffer);
+            Map<UUID, Set<String>> v = parseEntriesBySegment(buffer);
+            result.put(k, v);
+        }
+        return result;
+    }
+
+    private static Generation parseGeneration(ByteBuffer buffer) {
+        return new Generation(buffer.getInt(), 0, false);
+    }
+
+    private static Map<UUID, Set<String>> parseEntriesBySegment(ByteBuffer 
buffer) {
+        return parseEntriesBySegment(buffer.getInt(), buffer);
+    }
+
+    private static Map<UUID, Set<String>> parseEntriesBySegment(int count, 
ByteBuffer buffer) {
+        Map<UUID, Set<String>> result = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            UUID k = parseUUID(buffer);
+            Set<String> v = parseEntries(buffer);
+            result.put(k, v);
+        }
+        return result;
+    }
+
+    private static UUID parseUUID(ByteBuffer buffer) {
+        long msb = buffer.getLong();
+        long lsb = buffer.getLong();
+        return new UUID(msb, lsb);
+    }
+
+    private static Set<String> parseEntries(ByteBuffer buffer) {
+        return parseEntries(buffer.getInt(), buffer);
+    }
+
+    private static Set<String> parseEntries(int count, ByteBuffer buffer) {
+        Set<String> entries = new HashSet<>(count);
+        for (int i = 0; i < count; i++) {
+            entries.add(parseString(buffer));
+        }
+        return entries;
+    }
+
+    private static String parseString(ByteBuffer buffer) {
+        return parseString(buffer.getInt(), buffer);
+    }
+
+    private static String parseString(int length, ByteBuffer buffer) {
+        byte[] data = new byte[length];
+        buffer.get(data);
+        return new String(data, Charsets.UTF_8);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV1.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.CRC32;
+
+import com.google.common.base.Charsets;
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
+class BinaryReferencesIndexLoaderV2 {
+
+    static final int MAGIC = ('\n' << 24) + ('1' << 16) + ('B' << 8) + '\n';
+
+    private static final int FOOTER_SIZE = 16;
+
+    static BinaryReferencesIndex loadBinaryReferencesIndex(ReaderAtEnd reader) 
throws IOException, InvalidBinaryReferencesIndexException {
+        ByteBuffer meta = reader.readAtEnd(FOOTER_SIZE, FOOTER_SIZE);
+
+        int crc32 = meta.getInt();
+        int count = meta.getInt();
+        int size = meta.getInt();
+        int magic = meta.getInt();
+
+        if (magic != MAGIC) {
+            throw new InvalidBinaryReferencesIndexException("Invalid magic 
number");
+        }
+        if (count < 0) {
+            throw new InvalidBinaryReferencesIndexException("Invalid count");
+        }
+        if (size < count * 22 + 16) {
+            throw new InvalidBinaryReferencesIndexException("Invalid size");
+        }
+
+        ByteBuffer buffer = reader.readAtEnd(size, size - FOOTER_SIZE);
+
+        CRC32 checksum = new CRC32();
+        byte[] data = new byte[size - FOOTER_SIZE];
+        buffer.mark();
+        buffer.get(data);
+        buffer.reset();
+        checksum.update(data);
+
+        if ((int) (checksum.getValue()) != crc32) {
+            throw new InvalidBinaryReferencesIndexException("Invalid 
checksum");
+        }
+
+        return new BinaryReferencesIndex(parseBinaryReferencesIndex(count, 
buffer));
+    }
+
+    private static Map<Generation, Map<UUID, Set<String>>> 
parseBinaryReferencesIndex(int count, ByteBuffer buffer) {
+        Map<Generation, Map<UUID, Set<String>>> result = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            Generation k = parseGeneration(buffer);
+            Map<UUID, Set<String>> v = parseEntriesBySegment(buffer);
+            result.put(k, v);
+        }
+        return result;
+    }
+
+    private static Generation parseGeneration(ByteBuffer buffer) {
+        int generation = buffer.getInt();
+        int full = buffer.getInt();
+        boolean compacted = buffer.get() != 0;
+        return new Generation(generation, full, compacted);
+    }
+
+    private static Map<UUID, Set<String>> parseEntriesBySegment(ByteBuffer 
buffer) {
+        return parseEntriesBySegment(buffer.getInt(), buffer);
+    }
+
+    private static Map<UUID, Set<String>> parseEntriesBySegment(int count, 
ByteBuffer buffer) {
+        Map<UUID, Set<String>> result = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            UUID k = parseUUID(buffer);
+            Set<String> v = parseEntries(buffer);
+            result.put(k, v);
+        }
+        return result;
+    }
+
+    private static UUID parseUUID(ByteBuffer buffer) {
+        long msb = buffer.getLong();
+        long lsb = buffer.getLong();
+        return new UUID(msb, lsb);
+    }
+
+    private static Set<String> parseEntries(ByteBuffer buffer) {
+        return parseEntries(buffer.getInt(), buffer);
+    }
+
+    private static Set<String> parseEntries(int count, ByteBuffer buffer) {
+        Set<String> entries = new HashSet<>(count);
+        for (int i = 0; i < count; i++) {
+            entries.add(parseString(buffer));
+        }
+        return entries;
+    }
+
+    private static String parseString(ByteBuffer buffer) {
+        return parseString(buffer.getInt(), buffer);
+    }
+
+    private static String parseString(int length, ByteBuffer buffer) {
+        byte[] data = new byte[length];
+        buffer.get(data);
+        return new String(data, Charsets.UTF_8);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexLoaderV2.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.CRC32;
+
+import com.google.common.base.Charsets;
+
+/**
+ * Maintains the transient state of a binary references index, formats it and
+ * serializes it.
+ */
+public class BinaryReferencesIndexWriter {
+
+    /**
+     * Create a new, empty instance of {@link BinaryReferencesIndexWriter}.
+     *
+     * @return An instance of {@link BinaryReferencesIndexWriter}.
+     */
+    public static BinaryReferencesIndexWriter newBinaryReferencesIndexWriter() 
{
+        return new BinaryReferencesIndexWriter();
+    }
+
+    private final Map<Generation, Map<UUID, Set<String>>> entries;
+
+    private BinaryReferencesIndexWriter() {
+        entries = new HashMap<>();
+    }
+
+    /**
+     * Add an entry to the binary references index.
+     *
+     * @param generation The generation of the segment containing the
+     *                   reference.
+     * @param full       The full generation of the segment containing the
+     *                   reference.
+     * @param compacted  {@code true} if the segment containing the reference 
is
+     *                   created by a compaction operation.
+     * @param segment    The identifier of the segment containing the
+     *                   reference.
+     * @param reference  The binary reference.
+     */
+    public void addEntry(int generation, int full, boolean compacted, UUID 
segment, String reference) {
+        entries
+            .computeIfAbsent(new Generation(generation, full, compacted), k -> 
new HashMap<>())
+            .computeIfAbsent(segment, k -> new HashSet<>())
+            .add(reference);
+    }
+
+    /**
+     * Write the current state of this instance to an array of bytes.
+     *
+     * @return An array of bytes containing the serialized state of the binary
+     * references index.
+     */
+    public byte[] write() {
+        int binaryReferenceSize = 0;
+
+        // The following information are stored in the footer as meta-
+        // information about the entry.
+
+        // 4 bytes to store a magic number identifying this entry as containing
+        // references to binary values.
+        binaryReferenceSize += 4;
+
+        // 4 bytes to store the CRC32 checksum of the data in this entry.
+        binaryReferenceSize += 4;
+
+        // 4 bytes to store the length of this entry, without including the
+        // optional padding.
+        binaryReferenceSize += 4;
+
+        // 4 bytes to store the number of generations pairs in the binary
+        // references map.
+        binaryReferenceSize += 4;
+
+        // The following information are stored as part of the main content of
+        // this entry, after the optional padding.
+
+        for (Map<UUID, Set<String>> segmentToReferences : entries.values()) {
+            // 4 bytes per generation to store the full generation number.
+            binaryReferenceSize += 4;
+
+            // 4 bytes per generation to store the tail generation number.
+            binaryReferenceSize += 4;
+
+            // 1 byte per generation to store the "tail" flag.
+            binaryReferenceSize += 1;
+
+            // 4 bytes per generation to store the number of segments.
+            binaryReferenceSize += 4;
+
+            for (Set<String> references : segmentToReferences.values()) {
+                // 16 bytes per segment identifier.
+                binaryReferenceSize += 16;
+
+                // 4 bytes to store the number of references for this segment.
+                binaryReferenceSize += 4;
+
+                for (String reference : references) {
+                    // 4 bytes for each reference to store the length of the 
reference.
+                    binaryReferenceSize += 4;
+
+                    // A variable amount of bytes, depending on the reference 
itself.
+                    binaryReferenceSize += 
reference.getBytes(Charsets.UTF_8).length;
+                }
+            }
+        }
+
+        ByteBuffer buffer = ByteBuffer.allocate(binaryReferenceSize);
+
+        for (Entry<Generation, Map<UUID, Set<String>>> be : 
entries.entrySet()) {
+            Generation generation = be.getKey();
+            Map<UUID, Set<String>> segmentToReferences = be.getValue();
+
+            buffer.putInt(generation.generation);
+            buffer.putInt(generation.full);
+            buffer.put((byte) (generation.compacted ? 1 : 0));
+            buffer.putInt(segmentToReferences.size());
+
+            for (Entry<UUID, Set<String>> se : segmentToReferences.entrySet()) 
{
+                UUID segmentId = se.getKey();
+                Set<String> references = se.getValue();
+
+                buffer.putLong(segmentId.getMostSignificantBits());
+                buffer.putLong(segmentId.getLeastSignificantBits());
+                buffer.putInt(references.size());
+
+                for (String reference : references) {
+                    byte[] bytes = reference.getBytes(Charsets.UTF_8);
+
+                    buffer.putInt(bytes.length);
+                    buffer.put(bytes);
+                }
+            }
+        }
+
+        CRC32 checksum = new CRC32();
+        checksum.update(buffer.array(), 0, buffer.position());
+
+        buffer.putInt((int) checksum.getValue());
+        buffer.putInt(entries.size());
+        buffer.putInt(binaryReferenceSize);
+        buffer.putInt(BinaryReferencesIndexLoaderV2.MAGIC);
+
+        return buffer.array();
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/BinaryReferencesIndexWriter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+import java.util.Objects;
+
+class Generation {
+
+    final int generation;
+
+    final int full;
+
+    final boolean compacted;
+
+    Generation(int generation, int full, boolean compacted) {
+        this.generation = generation;
+        this.full = full;
+        this.compacted = compacted;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null) {
+            return false;
+        }
+        if (getClass() != o.getClass()) {
+            return false;
+        }
+        return equals((Generation) o);
+    }
+
+    private boolean equals(Generation that) {
+        return generation == that.generation && full == that.full && compacted 
== that.compacted;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(generation, full, compacted);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/Generation.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java?rev=1804413&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java
 Tue Aug  8 12:50:42 2017
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.binaries;
+
+public class InvalidBinaryReferencesIndexException extends Exception {
+
+    InvalidBinaryReferencesIndexException(String message) {
+        super(message);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/binaries/InvalidBinaryReferencesIndexException.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
 Tue Aug  8 12:50:42 2017
@@ -21,6 +21,8 @@ import static com.google.common.base.Pre
 
 import java.io.IOException;
 
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
 /**
  * Load and validate the index of a TAR file.
  */

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV1.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV1.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV1.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV1.java
 Tue Aug  8 12:50:42 2017
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.zip.CRC32;
 
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
 class IndexLoaderV1 {
 
     static final int MAGIC = ('\n' << 24) + ('0' << 16) + ('K' << 8) + '\n';

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
 Tue Aug  8 12:50:42 2017
@@ -23,6 +23,8 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.zip.CRC32;
 
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+
 class IndexLoaderV2 {
 
     static final int MAGIC = ('\n' << 24) + ('1' << 16) + ('K' << 8) + '\n';

Copied: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/ReaderAtEnd.java
 (from r1804412, 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/ReaderAtEnd.java)
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/ReaderAtEnd.java?p2=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/ReaderAtEnd.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/ReaderAtEnd.java&r1=1804412&r2=1804413&rev=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/ReaderAtEnd.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/util/ReaderAtEnd.java
 Tue Aug  8 12:50:42 2017
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.jackrabbit.oak.segment.file.tar.index;
+package org.apache.jackrabbit.oak.segment.util;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java?rev=1804413&r1=1804412&r2=1804413&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
 Tue Aug  8 12:50:42 2017
@@ -21,14 +21,15 @@ package org.apache.jackrabbit.oak.segmen
 import static com.google.common.base.Charsets.UTF_8;
 import static com.google.common.collect.Lists.newArrayList;
 import static com.google.common.collect.Maps.newHashMap;
+import static 
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 import static org.mockito.internal.util.collections.Sets.newSet;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -43,7 +44,7 @@ import org.junit.rules.TemporaryFolder;
 public class TarFileTest {
 
     private static GCGeneration generation(int full) {
-        return GCGeneration.newGCGeneration(full, 0, false);
+        return newGCGeneration(full, 0, false);
     }
 
     private File file;
@@ -84,12 +85,12 @@ public class TarFileTest {
         byte[] buffer = data.getBytes(UTF_8);
 
         try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
-            writer.writeEntry(msb, lsb, buffer, 0, buffer.length, 
GCGeneration.newGCGeneration(1, 2, false));
+            writer.writeEntry(msb, lsb, buffer, 0, buffer.length, 
newGCGeneration(1, 2, false));
         }
 
         try (TarReader reader = TarReader.open(file, false, new 
IOMonitorAdapter())) {
             TarEntry[] entries = reader.getEntries();
-            assertEquals(GCGeneration.newGCGeneration(1, 2, false), 
entries[0].generation());
+            assertEquals(newGCGeneration(1, 2, false), 
entries[0].generation());
         }
     }
 
@@ -102,12 +103,12 @@ public class TarFileTest {
         byte[] buffer = data.getBytes(UTF_8);
 
         try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
-            writer.writeEntry(msb, lsb, buffer, 0, buffer.length, 
GCGeneration.newGCGeneration(1, 2, true));
+            writer.writeEntry(msb, lsb, buffer, 0, buffer.length, 
newGCGeneration(1, 2, true));
         }
 
         try (TarReader reader = TarReader.open(file, false, new 
IOMonitorAdapter())) {
             TarEntry[] entries = reader.getEntries();
-            assertEquals(GCGeneration.newGCGeneration(1, 2, true), 
entries[0].generation());
+            assertEquals(newGCGeneration(1, 2, true), entries[0].generation());
         }
     }
 
@@ -154,7 +155,16 @@ public class TarFileTest {
         expected.put(generation(3), three);
 
         try (TarReader reader = TarReader.open(file, false, new 
IOMonitorAdapter())) {
-            assertEquals(expected, reader.getBinaryReferences());
+            Map<GCGeneration, Map<UUID, Set<String>>> actual = new HashMap<>();
+
+            reader.getBinaryReferences().forEach((generation, full, compacted, 
id, reference) -> {
+                actual
+                    .computeIfAbsent(newGCGeneration(generation, full, 
compacted), x -> new HashMap<>())
+                    .computeIfAbsent(id, x -> new HashSet<>())
+                    .add(reference);
+            });
+
+            assertEquals(expected, actual);
         }
     }
 
@@ -189,7 +199,15 @@ public class TarFileTest {
                 references.put(generation(1), one);
                 references.put(generation(2), two);
 
-                assertEquals(references, swept.getBinaryReferences());
+                Map<GCGeneration, Map<UUID, Set<String>>> actual = new 
HashMap<>();
+                swept.getBinaryReferences().forEach((generation, full, 
compacted, uuid, reference) -> {
+                    actual
+                        .computeIfAbsent(newGCGeneration(generation, full, 
compacted), x -> new HashMap<>())
+                        .computeIfAbsent(uuid, x -> new HashSet<>())
+                        .add(reference);
+                });
+
+                assertEquals(references, actual);
             }
         }
     }
@@ -198,14 +216,18 @@ public class TarFileTest {
     public void binaryReferencesIndexShouldContainCompleteGCGeneration() 
throws Exception {
         try (TarWriter writer = new TarWriter(file, new IOMonitorAdapter())) {
             writer.writeEntry(0x00, 0x00, new byte[] {0x01, 0x02, 0x3}, 0, 3, 
generation(0));
-            writer.addBinaryReference(GCGeneration.newGCGeneration(1, 2, 
false), new UUID(1, 2), "r1");
-            writer.addBinaryReference(GCGeneration.newGCGeneration(3, 4, 
true), new UUID(3, 4), "r2");
+            writer.addBinaryReference(newGCGeneration(1, 2, false), new 
UUID(1, 2), "r1");
+            writer.addBinaryReference(newGCGeneration(3, 4, true), new UUID(3, 
4), "r2");
         }
         try (TarReader reader = TarReader.open(file, false, new 
IOMonitorAdapter())) {
             Set<GCGeneration> expected = new HashSet<>();
-            expected.add(GCGeneration.newGCGeneration(1, 2, false));
-            expected.add(GCGeneration.newGCGeneration(3, 4, true));
-            assertEquals(expected, reader.getBinaryReferences().keySet());
+            expected.add(newGCGeneration(1, 2, false));
+            expected.add(newGCGeneration(3, 4, true));
+            Set<GCGeneration> actual = new HashSet<>();
+            reader.getBinaryReferences().forEach((generation, full, compacted, 
segment, reference) -> {
+                actual.add(newGCGeneration(generation, full, compacted));
+            });
+            assertEquals(expected, actual);
         }
     }
 


Reply via email to