Author: frm
Date: Thu Aug 10 15:00:43 2017
New Revision: 1804675

URL: http://svn.apache.org/viewvc?rev=1804675&view=rev
Log:
OAK-6457 - Encapsulate access to the buffers containing segment data

Added:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java
   (with props)
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java
   (with props)
Modified:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1804675&r1=1804674&r2=1804675&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
 Thu Aug 10 15:00:43 2017
@@ -20,7 +20,6 @@ package org.apache.jackrabbit.oak.segmen
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Preconditions.checkPositionIndexes;
 import static com.google.common.base.Preconditions.checkState;
 import static java.util.Arrays.fill;
 import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
@@ -30,6 +29,7 @@ import static org.apache.jackrabbit.oak.
 import static org.apache.jackrabbit.oak.segment.SegmentStream.BLOCK_SIZE;
 import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
 import static org.apache.jackrabbit.oak.segment.SegmentVersion.isValid;
+import static 
org.apache.jackrabbit.oak.segment.data.SegmentData.newSegmentData;
 import static 
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
 
 import java.io.IOException;
@@ -37,8 +37,6 @@ import java.io.OutputStream;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.nio.ByteBuffer;
-import java.nio.channels.Channels;
-import java.nio.channels.WritableByteChannel;
 import java.util.Arrays;
 import java.util.Iterator;
 import java.util.UUID;
@@ -50,11 +48,15 @@ import com.google.common.base.Charsets;
 import com.google.common.collect.AbstractIterator;
 import org.apache.commons.io.HexDump;
 import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.io.output.WriterOutputStream;
 import org.apache.jackrabbit.oak.api.PropertyState;
 import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.commons.StringUtils;
 import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
 import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry;
+import org.apache.jackrabbit.oak.segment.data.RecordIdData;
+import org.apache.jackrabbit.oak.segment.data.SegmentData;
+import org.apache.jackrabbit.oak.segment.data.StringData;
 import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
 
 /**
@@ -137,8 +139,7 @@ public class Segment {
     @Nonnull
     private final SegmentId id;
 
-    @Nonnull
-    private final ByteBuffer data;
+    private final SegmentData data;
 
     /**
      * Version of the segment storage format.
@@ -174,19 +175,16 @@ public class Segment {
                    @Nonnull final ByteBuffer data) {
         this.reader = checkNotNull(reader);
         this.id = checkNotNull(id);
-        this.data = checkNotNull(data);
+        this.data = newSegmentData(checkNotNull(data).slice());
         if (id.isDataSegmentId()) {
-            byte segmentVersion = data.get(3);
-            checkState(data.get(0) == '0'
-                    && data.get(1) == 'a'
-                    && data.get(2) == 'K'
-                    && isValid(segmentVersion),
-                new Object() {  // Defer evaluation of error message
+            byte segmentVersion = this.data.getVersion();
+            checkState(this.data.getSignature().equals("0aK") && 
isValid(segmentVersion), new Object() {
+
                     @Override
                     public String toString() {
-                        return "Invalid segment format. Dumping segment " + id 
+ "\n"
-                            + toHex(data.array());
+                        return String.format("Invalid segment format. Dumping 
segment %s\n%s", id, toHex(data.array()));
                     }
+
             });
             this.version = SegmentVersion.fromByte(segmentVersion);
             this.recordNumbers = readRecordNumberOffsets();
@@ -216,38 +214,31 @@ public class Segment {
      * @return An instance of {@link RecordNumbers}, never {@code null}.
      */
     private RecordNumbers readRecordNumberOffsets() {
-        int recordNumberCount = getRecordNumberCount();
+        int recordNumberCount = data.getRecordReferencesCount();
+
         if (recordNumberCount == 0) {
             return EMPTY_RECORD_NUMBERS;
         }
 
-        int position = HEADER_SIZE + data.position()
-                + getReferencedSegmentIdCount() * SEGMENT_REFERENCE_SIZE;
-        int maxIndex = data.getInt(position + (recordNumberCount - 1) * 9);
+        int maxIndex = data.getRecordReferenceNumber(recordNumberCount - 1);
 
         byte[] types = new byte[maxIndex + 1];
         int[] offsets = new int[maxIndex + 1];
         fill(offsets, -1);
 
         for (int i = 0; i < recordNumberCount; i++) {
-            int recordNumber = data.getInt(position);
-            position += 4;
-            types[recordNumber] = data.get(position);
-            position += 1;
-            offsets[recordNumber] = data.getInt(position);
-            position += 4;
+            int recordNumber = data.getRecordReferenceNumber(i);
+            types[recordNumber] = data.getRecordReferenceType(i);
+            offsets[recordNumber] = data.getRecordReferenceOffset(i);
         }
 
         return new ImmutableRecordNumbers(offsets, types);
     }
 
-    private SegmentReferences readReferencedSegments(
-            final SegmentIdProvider idProvider) {
-        checkState(getReferencedSegmentIdCount() + 1 < 0xffff,
-                "Segment cannot have more than 0xffff references");
+    private SegmentReferences readReferencedSegments(final SegmentIdProvider 
idProvider) {
+        checkState(getReferencedSegmentIdCount() + 1 < 0xffff, "Segment cannot 
have more than 0xffff references");
 
         final int referencedSegmentIdCount = getReferencedSegmentIdCount();
-        final int refOffset = data.position() + HEADER_SIZE;
 
         // We need to keep SegmentId references (as opposed to e.g. UUIDs)
         // here as frequently resolving the segment ids via the segment id
@@ -257,19 +248,21 @@ public class Segment {
         // is managed via the SegmentCache. It is the size of that cache that
         // keeps overall heap usage by Segment instances bounded.
         // See OAK-6106.
+
         final SegmentId[] refIds = new SegmentId[referencedSegmentIdCount];
+
         return new SegmentReferences() {
+
             @Override
             public SegmentId getSegmentId(int reference) {
                 checkArgument(reference <= referencedSegmentIdCount, "Segment 
reference out of bounds");
                 SegmentId id = refIds[reference - 1];
                 if (id == null) {
-                    synchronized(refIds) {
+                    synchronized (refIds) {
                         id = refIds[reference - 1];
                         if (id == null) {
-                            int position = refOffset + (reference - 1) * 
SEGMENT_REFERENCE_SIZE;
-                            long msb = data.getLong(position);
-                            long lsb = data.getLong(position + 8);
+                            long msb = data.getSegmentReferenceMsb(reference - 
1);
+                            long lsb = data.getSegmentReferenceLsb(reference - 
1);
                             id = idProvider.newSegmentId(msb, lsb);
                             refIds[reference - 1] = id;
                         }
@@ -282,7 +275,9 @@ public class Segment {
             @Override
             public Iterator<SegmentId> iterator() {
                 return new AbstractIterator<SegmentId>() {
+
                     private int reference = 1;
+
                     @Override
                     protected SegmentId computeNext() {
                         if (reference <= referencedSegmentIdCount) {
@@ -291,22 +286,25 @@ public class Segment {
                             return endOfData();
                         }
                     }
+
                 };
             }
+
         };
     }
 
-    Segment(@Nonnull SegmentId id,
-            @Nonnull SegmentReader reader,
-            @Nonnull byte[] buffer,
-            @Nonnull RecordNumbers recordNumbers,
-            @Nonnull SegmentReferences segmentReferences,
-            @Nonnull String info
+    Segment(
+        @Nonnull SegmentId id,
+        @Nonnull SegmentReader reader,
+        @Nonnull byte[] buffer,
+        @Nonnull RecordNumbers recordNumbers,
+        @Nonnull SegmentReferences segmentReferences,
+        @Nonnull String info
     ) {
         this.id = checkNotNull(id);
         this.reader = checkNotNull(reader);
         this.info = checkNotNull(info);
-        this.data = ByteBuffer.wrap(checkNotNull(buffer));
+        this.data = newSegmentData(buffer);
         this.version = SegmentVersion.fromByte(buffer[3]);
         this.recordNumbers = recordNumbers;
         this.segmentReferences = segmentReferences;
@@ -317,50 +315,16 @@ public class Segment {
         return version;
     }
 
-    private int pos(int recordNumber, int length) {
-        return pos(recordNumber, 0, 0, length);
-    }
-
-    private int pos(int recordNumber, int rawOffset, int length) {
-        return pos(recordNumber, rawOffset, 0, length);
-    }
-
-    /**
-     * Maps the given record number to the respective position within the
-     * internal {@link #data} array. The validity of a record with the given
-     * length at the given record number is also verified.
-     *
-     * @param recordNumber   record number
-     * @param rawOffset      offset to add to the base position of the record
-     * @param recordIdOffset offset to add to to the base position of the
-     *                       record, multiplied by the length of a record ID
-     * @param length         record length
-     * @return position within the data array
-     */
-    private int pos(int recordNumber, int rawOffset, int recordIdOffset, int 
length) {
-        int offset = recordNumbers.getOffset(recordNumber);
-
-        if (offset == -1) {
-            throw new IllegalStateException("invalid record number");
-        }
-
-        int base = offset + rawOffset + recordIdOffset * RECORD_ID_BYTES;
-        checkPositionIndexes(base, base + length, MAX_SEGMENT_SIZE);
-        int pos = data.limit() - MAX_SEGMENT_SIZE + base;
-        checkState(pos >= data.position());
-        return pos;
-    }
-
     public SegmentId getSegmentId() {
         return id;
     }
 
     public int getReferencedSegmentIdCount() {
-        return data.getInt(REFERENCED_SEGMENT_ID_COUNT_OFFSET);
+        return data.getSegmentReferencesCount();
     }
 
     private int getRecordNumberCount() {
-        return data.getInt(RECORD_NUMBER_COUNT_OFFSET);
+        return data.getRecordReferencesCount();
     }
 
     public UUID getReferencedSegmentId(int index) {
@@ -368,22 +332,19 @@ public class Segment {
     }
 
     /**
-     * Determine the gc generation a segment from its data. Note that bulk 
segments don't have
-     * generations (i.e. stay at 0).
+     * Determine the gc generation a segment from its data. Note that bulk
+     * segments don't have generations (i.e. stay at 0).
      *
-     * @param data         the date of the segment
-     * @param segmentId    the id of the segment
-     * @return  the gc generation of this segment or {@link GCGeneration#NULL} 
if this is bulk segment.
+     * @param data      the data of the segment
+     * @param segmentId the id of the segment
+     * @return the gc generation of this segment or {@link GCGeneration#NULL} 
if
+     * this is bulk segment.
      */
-    @Nonnull
-    public static GCGeneration getGcGeneration(ByteBuffer data, UUID 
segmentId) {
+    public static GCGeneration getGcGeneration(SegmentData data, UUID 
segmentId) {
         if (isDataSegmentId(segmentId.getLeastSignificantBits())) {
-            int generation = data.getInt(GC_GENERATION_OFFSET);
-            int fullGeneration = data.getInt(GC_FULL_GENERATION_OFFSET);
-            return newGCGeneration(generation, fullGeneration & 0x7fffffff, 
fullGeneration < 0);
-        } else {
-            return GCGeneration.NULL;
+            return newGCGeneration(data.getGeneration(), 
data.getFullGeneration(), data.isCompacted());
         }
+        return GCGeneration.NULL;
     }
 
     /**
@@ -422,48 +383,46 @@ public class Segment {
     }
 
     public int size() {
-        return data.remaining();
+        return data.size();
     }
 
     byte readByte(int recordNumber) {
-        return readByte(recordNumber, 0);
+        return data.readByte(recordNumbers.getOffset(recordNumber));
     }
 
     byte readByte(int recordNumber, int offset) {
-        return data.get(pos(recordNumber, offset, 1));
+        return data.readByte(recordNumbers.getOffset(recordNumber) + offset);
     }
 
     short readShort(int recordNumber) {
-        return data.getShort(pos(recordNumber, 2));
+        return data.readShort(recordNumbers.getOffset(recordNumber));
     }
 
     int readInt(int recordNumber) {
-        return data.getInt(pos(recordNumber, 4));
+        return data.readInt(recordNumbers.getOffset(recordNumber));
     }
 
     int readInt(int recordNumber, int offset) {
-        return data.getInt(pos(recordNumber, offset, 4));
+        return data.readInt(recordNumbers.getOffset(recordNumber) + offset);
     }
 
     long readLong(int recordNumber) {
-        return data.getLong(pos(recordNumber, 8));
+        return data.readLong(recordNumbers.getOffset(recordNumber));
     }
 
     void readBytes(int recordNumber, int position, byte[] buffer, int offset, 
int length) {
-        checkNotNull(buffer);
-        checkPositionIndexes(offset, offset + length, buffer.length);
-        ByteBuffer d = readBytes(recordNumber, position, length);
-        d.get(buffer, offset, length);
+        readBytes(recordNumber, position, length).get(buffer, offset, length);
     }
 
     ByteBuffer readBytes(int recordNumber, int position, int length) {
-        int pos = pos(recordNumber, position, length);
-        return slice(pos, length);
+        return data.readBytes(recordNumbers.getOffset(recordNumber) + 
position, length);
     }
 
     @Nonnull
     RecordId readRecordId(int recordNumber, int rawOffset, int recordIdOffset) 
{
-        return internalReadRecordId(pos(recordNumber, rawOffset, 
recordIdOffset, RECORD_ID_BYTES));
+        int offset = recordNumbers.getOffset(recordNumber) + rawOffset + 
recordIdOffset * RecordIdData.BYTES;
+        RecordIdData recordIdData = data.readRecordId(offset);
+        return new 
RecordId(dereferenceSegmentId(recordIdData.getSegmentReference()), 
recordIdData.getRecordNumber());
     }
 
     RecordId readRecordId(int recordNumber, int rawOffset) {
@@ -475,16 +434,6 @@ public class Segment {
     }
 
     @Nonnull
-    private RecordId internalReadRecordId(int pos) {
-        SegmentId segmentId = 
dereferenceSegmentId(asUnsigned(data.getShort(pos)));
-        return new RecordId(segmentId, data.getInt(pos + 2));
-    }
-
-    private static int asUnsigned(short value) {
-        return value & 0xffff;
-    }
-
-    @Nonnull
     private SegmentId dereferenceSegmentId(int reference) {
         if (reference == 0) {
             return id;
@@ -500,29 +449,23 @@ public class Segment {
     }
 
     @Nonnull
-    String readString(int offset) {
-        int pos = pos(offset, 1);
-        long length = internalReadLength(pos);
-        if (length < SMALL_LIMIT) {
-            return Charsets.UTF_8.decode(slice(pos + 1, (int) 
length)).toString();
-        } else if (length < MEDIUM_LIMIT) {
-            return Charsets.UTF_8.decode(slice(pos + 2, (int) 
length)).toString();
-        } else if (length < Integer.MAX_VALUE) {
-            int size = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
-            ListRecord list = new ListRecord(internalReadRecordId(pos + 8), 
size);
-            try (SegmentStream stream = new SegmentStream(new RecordId(id, 
offset), list, length)) {
+    String readString(int recordNumber) {
+        StringData data = 
this.data.readString(recordNumbers.getOffset(recordNumber));
+
+        if (data.isString()) {
+            return data.getString();
+        }
+
+        if (data.isRecordId()) {
+            SegmentId segmentId = 
dereferenceSegmentId(data.getRecordId().getSegmentReference());
+            RecordId recordId = new RecordId(segmentId, 
data.getRecordId().getRecordNumber());
+            ListRecord list = new ListRecord(recordId, (data.getLength() + 
BLOCK_SIZE - 1) / BLOCK_SIZE);
+            try (SegmentStream stream = new SegmentStream(new RecordId(id, 
recordNumber), list, data.getLength())) {
                 return stream.getString();
             }
-        } else {
-            throw new IllegalStateException("String is too long: " + length);
         }
-    }
 
-    private ByteBuffer slice(int pos, int length) {
-        ByteBuffer buffer = data.duplicate();
-        buffer.position(pos);
-        buffer.limit(pos + length);
-        return buffer.slice();
+        throw new IllegalStateException("Invalid return value");
     }
 
     @Nonnull
@@ -592,28 +535,7 @@ public class Segment {
     }
 
     long readLength(int recordNumber) {
-        return internalReadLength(pos(recordNumber, 1));
-    }
-
-    private long internalReadLength(int pos) {
-        int length = data.get(pos++) & 0xff;
-        if ((length & 0x80) == 0) {
-            return length;
-        } else if ((length & 0x40) == 0) {
-            return ((length & 0x3f) << 8
-                    | data.get(pos) & 0xff)
-                    + SMALL_LIMIT;
-        } else {
-            return (((long) length & 0x3f) << 56
-                    | ((long) (data.get(pos++) & 0xff)) << 48
-                    | ((long) (data.get(pos++) & 0xff)) << 40
-                    | ((long) (data.get(pos++) & 0xff)) << 32
-                    | ((long) (data.get(pos++) & 0xff)) << 24
-                    | ((long) (data.get(pos++) & 0xff)) << 16
-                    | ((long) (data.get(pos++) & 0xff)) << 8
-                    | ((long) (data.get(pos) & 0xff)))
-                    + MEDIUM_LIMIT;
-        }
+        return data.readLength(recordNumbers.getOffset(recordNumber));
     }
 
     //------------------------------------------------------------< Object >--
@@ -622,69 +544,34 @@ public class Segment {
     public String toString() {
         StringWriter string = new StringWriter();
         try (PrintWriter writer = new PrintWriter(string)) {
-            int length = data.remaining();
-
-            writer.format("Segment %s (%d bytes)%n", id, length);
+            writer.format("Segment %s (%d bytes)%n", id, data.size());
             String segmentInfo = getSegmentInfo();
             if (segmentInfo != null) {
                 writer.format("Info: %s, Generation: %s%n", segmentInfo, 
getGcGeneration());
             }
             if (id.isDataSegmentId()) {
                 
writer.println("--------------------------------------------------------------------------");
-
                 int i = 1;
-
                 for (SegmentId segmentId : segmentReferences) {
                     writer.format("reference %02x: %s%n", i++, segmentId);
                 }
-
                 for (Entry entry : recordNumbers) {
-                    writer.format("%10s record %08x: %08x%n",
-                            entry.getType(), entry.getRecordNumber(), 
entry.getOffset());
+                    writer.format("%10s record %08x: %08x%n", entry.getType(), 
entry.getRecordNumber(), entry.getOffset());
                 }
             }
             
writer.println("--------------------------------------------------------------------------");
-            int pos = data.limit() - ((length + 15) & ~15);
-            while (pos < data.limit()) {
-                writer.format("%04x: ", (MAX_SEGMENT_SIZE - data.limit() + 
pos) >> RECORD_ALIGN_BITS);
-                for (int i = 0; i < 16; i++) {
-                    if (i > 0 && i % 4 == 0) {
-                        writer.append(' ');
-                    }
-                    if (pos + i >= data.position()) {
-                        byte b = data.get(pos + i);
-                        writer.format("%02x ", b & 0xff);
-                    } else {
-                        writer.append("   ");
-                    }
-                }
-                writer.append(' ');
-                for (int i = 0; i < 16; i++) {
-                    if (pos + i >= data.position()) {
-                        byte b = data.get(pos + i);
-                        if (b >= ' ' && b < 127) {
-                            writer.append((char) b);
-                        } else {
-                            writer.append('.');
-                        }
-                    } else {
-                        writer.append(' ');
-                    }
-                }
-                writer.println();
-                pos += 16;
+            try {
+                data.hexDump(new WriterOutputStream(writer, Charsets.UTF_8));
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
             }
             
writer.println("--------------------------------------------------------------------------");
-            return string.toString();
         }
+        return string.toString();
     }
 
     public void writeTo(OutputStream stream) throws IOException {
-        ByteBuffer buffer = data.duplicate();
-        WritableByteChannel channel = Channels.newChannel(stream);
-        while (buffer.hasRemaining()) {
-            channel.write(buffer);
-        }
+        data.binDump(stream);
     }
 
     /**
@@ -731,11 +618,9 @@ public class Segment {
 
             size += StringUtils.estimateMemoryUsage(info);
         }
-        if (!data.isDirect()) {
-            // seems to over report by 100+ bytes
-            size += size();
-        }
+        size += data.estimateMemoryUsage();
         size += id.estimateMemoryUsage();
         return size;
     }
+
 }

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1804675&r1=1804674&r2=1804675&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
 Thu Aug 10 15:00:43 2017
@@ -45,6 +45,7 @@ import javax.annotation.CheckForNull;
 import javax.annotation.Nonnull;
 
 import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry;
+import org.apache.jackrabbit.oak.segment.data.SegmentData;
 import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java?rev=1804675&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java
 Thu Aug 10 15:00:43 2017
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.data;
+
+public class RecordIdData {
+
+    public static final int BYTES = Short.BYTES + Integer.BYTES;
+
+    private final int segmentReference;
+
+    private final int recordNumber;
+
+    RecordIdData(int segmentReference, int recordNumber) {
+        this.segmentReference = segmentReference;
+        this.recordNumber = recordNumber;
+    }
+
+    public int getSegmentReference() {
+        return segmentReference;
+    }
+
+    public int getRecordNumber() {
+        return recordNumber;
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/RecordIdData.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java?rev=1804675&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java
 Thu Aug 10 15:00:43 2017
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.data;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.HexDump;
+
+/**
+ * Access the data of a segment.
+ * <p>
+ * A segment is composed of a header and the proper data. The header has a
+ * fixed- and a variable-length part.
+ * <p>
+ * The fixed-length part of the header contains a {@link #getSignature()
+ * signature}, which is a string that uniquely identifies the rest of the
+ * content as a segment; a {@link #getVersion()}, which specifies the version 
of
+ * the binary format used to serialize the content of the segment; a {@link
+ * #getFullGeneration() full generation}, which describes the generation of the
+ * segment with regards to full compaction; a {@link #getGeneration()
+ * generation}, which identifies the generation of the segment with regards to
+ * full or tail compaction; a {@link #isCompacted() compacted flag}, which
+ * determines if the segment was written by a compaction operation; the {@link
+ * #getRecordReferencesCount() number of record references}, which is the 
number
+ * of record entries in the segment; the {@link #getSegmentReferencesCount()
+ * number of segment references}, which is the number of identifiers of other
+ * segments used by this segment.
+ * <p>
+ * The variable part of the header contains the list of segment references and
+ * the list of record references. A segment references is composed by the 
{@link
+ * #getSegmentReferenceMsb(int) most significant bits} and {@link
+ * #getSegmentReferenceLsb(int) lsb} of the segment identifier. A record
+ * reference is composed of a {@link #getRecordReferenceNumber(int) record
+ * number}, a {@link #getRecordReferenceType(int) record type} and a {@link
+ * #getRecordReferenceOffset(int) record offset}.
+ * <p>
+ * The most prominent use for a segment is to hold record data. Many methods of
+ * this class allows access to the record data. These methods accept an integer
+ * representing an absolute position pointing to the record data. The absolute
+ * position, though, is virtual: it is computed on a virtual segment 256K long.
+ * This offset is usually obtained by accessing the {@link
+ * #getRecordReferenceOffset(int) record offset} of a record reference entry.
+ * The class will normalize the offset for the actual size of the segment, 
which
+ * can be smaller than 256K. It is acceptable to displace the offset of a 
record
+ * reference entry by a positive amount. This can be useful to access a field 
of
+ * a composite record saved at a specific offset.
+ */
+public class SegmentData {
+
+    private static final int HEADER_SIZE = 32;
+
+    private static final int SIGNATURE_OFFSET = 0;
+
+    private static final int SIGNATURE_LENGTH = 3;
+
+    private static final int VERSION_OFFSET = 3;
+
+    private static final int FULL_GENERATION_OFFSET = 4;
+
+    private static final int GENERATION_OFFSET = 10;
+
+    private static final int SEGMENT_REFERENCES_COUNT_OFFSET = 14;
+
+    private static final int SEGMENT_REFERENCE_LENGTH = 16;
+
+    private static final int RECORD_REFERENCES_COUNT_OFFSET = 18;
+
+    private static final int RECORD_REFERENCE_LENGTH = 9;
+
+    // Relative to a segment reference - BEGIN
+
+    private static final int SEGMENT_REFERENCE_MSB_OFFSET = 0;
+
+    private static final int SEGMENT_REFERENCE_LSB_OFFSET = 8;
+
+    // Relative to a segment reference - END
+
+    // Relative to a record reference - BEGIN
+
+    private static final int RECORD_REFERENCE_NUMBER_OFFSET = 0;
+
+    private static final int RECORD_REFERENCE_TYPE_OFFSET = 4;
+
+    private static final int RECORD_REFERENCE_OFFSET_OFFSET = 5;
+
+    // Relative to a record reference - END
+
+    private static final int MAX_SMALL_LENGTH_VALUE = 1 << 7;
+
+    private static final int MAX_MEDIUM_LENGTH_VALUE = (1 << 14) + 
MAX_SMALL_LENGTH_VALUE;
+
+    private static final int MAX_SEGMENT_SIZE = 1 << 18;
+
+    public static SegmentData newSegmentData(byte[] buffer) {
+        return new SegmentData(ByteBuffer.wrap(buffer));
+    }
+
+    public static SegmentData newSegmentData(ByteBuffer buffer) {
+        return new SegmentData(buffer);
+    }
+
+    private SegmentData(ByteBuffer buffer) {
+        this.buffer = buffer;
+    }
+
+    private final ByteBuffer buffer;
+
+    public byte getVersion() {
+        return buffer.get(VERSION_OFFSET);
+    }
+
+    public String getSignature() {
+        byte[] signature = new byte[SIGNATURE_LENGTH];
+
+        for (int i = 0; i < SIGNATURE_LENGTH; i++) {
+            signature[i] = buffer.get(SIGNATURE_OFFSET + i);
+        }
+
+        return new String(signature, Charsets.UTF_8);
+    }
+
+    public int getFullGeneration() {
+        return buffer.getInt(FULL_GENERATION_OFFSET) & 0x7fffffff;
+    }
+
+    public boolean isCompacted() {
+        return buffer.getInt(FULL_GENERATION_OFFSET) < 0;
+    }
+
+    public int getGeneration() {
+        return buffer.getInt(GENERATION_OFFSET);
+    }
+
+    public int getSegmentReferencesCount() {
+        return buffer.getInt(SEGMENT_REFERENCES_COUNT_OFFSET);
+    }
+
+    public int getRecordReferencesCount() {
+        return buffer.getInt(RECORD_REFERENCES_COUNT_OFFSET);
+    }
+
+    private int getRecordReferenceBase(int i) {
+        return HEADER_SIZE + getSegmentReferencesCount() * 
SEGMENT_REFERENCE_LENGTH + i * RECORD_REFERENCE_LENGTH;
+    }
+
+    public int getRecordReferenceNumber(int i) {
+        return buffer.getInt(getRecordReferenceBase(i) + 
RECORD_REFERENCE_NUMBER_OFFSET);
+    }
+
+    public byte getRecordReferenceType(int i) {
+        return buffer.get(getRecordReferenceBase(i) + 
RECORD_REFERENCE_TYPE_OFFSET);
+    }
+
+    public int getRecordReferenceOffset(int i) {
+        return buffer.getInt(getRecordReferenceBase(i) + 
RECORD_REFERENCE_OFFSET_OFFSET);
+    }
+
+    private int getSegmentReferenceBase(int i) {
+        return HEADER_SIZE + i * SEGMENT_REFERENCE_LENGTH;
+    }
+
+    public long getSegmentReferenceMsb(int i) {
+        return buffer.getLong(getSegmentReferenceBase(i) + 
SEGMENT_REFERENCE_MSB_OFFSET);
+    }
+
+    public long getSegmentReferenceLsb(int i) {
+        return buffer.getLong(getSegmentReferenceBase(i) + 
SEGMENT_REFERENCE_LSB_OFFSET);
+    }
+
+    private int index(int recordReferenceOffset) {
+        return buffer.limit() - (MAX_SEGMENT_SIZE - recordReferenceOffset);
+    }
+
+    public long readLength(int recordReferenceOffset) {
+        return internalReadLength(index(recordReferenceOffset));
+    }
+
+    private long internalReadLength(int index) {
+        int head = buffer.get(index) & 0xff;
+
+        if ((head & 0x80) == 0) {
+            return head;
+        }
+
+        if ((head & 0x40) == 0) {
+            return MAX_SMALL_LENGTH_VALUE + (buffer.getShort(index) & 0x3fff);
+        }
+
+        return MAX_MEDIUM_LENGTH_VALUE + (buffer.getLong(index) & 
0x3fffffffffffffffL);
+    }
+
+    public StringData readString(int recordReferenceOffset) {
+        return internalReadString(index(recordReferenceOffset));
+    }
+
+    private StringData internalReadString(int index) {
+        long length = internalReadLength(index);
+
+        if (length < MAX_SMALL_LENGTH_VALUE) {
+            return internalReadString(index + Byte.BYTES, (int) length);
+        }
+
+        if (length < MAX_MEDIUM_LENGTH_VALUE) {
+            return internalReadString(index + Short.BYTES, (int) length);
+        }
+
+        if (length < Integer.MAX_VALUE) {
+            return new StringData(internalReadRecordId(index + Long.BYTES), 
(int) length);
+        }
+
+        throw new IllegalStateException("String is too long: " + length);
+    }
+
+    private StringData internalReadString(int index, int length) {
+        ByteBuffer duplicate = buffer.duplicate();
+        duplicate.position(index);
+        duplicate.limit(index + length);
+        String string = Charsets.UTF_8.decode(duplicate).toString();
+        return new StringData(string, length);
+    }
+
+    public RecordIdData readRecordId(int recordReferenceOffset) {
+        return internalReadRecordId(index(recordReferenceOffset));
+    }
+
+    private RecordIdData internalReadRecordId(int index) {
+        int segmentReference = buffer.getShort(index) & 0xffff;
+        int recordNumber = buffer.getInt(index + Short.BYTES);
+        return new RecordIdData(segmentReference, recordNumber);
+    }
+
+    public byte readByte(int recordReferenceOffset) {
+        return buffer.get(index(recordReferenceOffset));
+    }
+
+    public int readInt(int recordReferenceOffset) {
+        return buffer.getInt(index(recordReferenceOffset));
+    }
+
+    public short readShort(int recordReferenceOffset) {
+        return buffer.getShort(index(recordReferenceOffset));
+    }
+
+    public long readLong(int recordReferenceOffset) {
+        return buffer.getLong(index(recordReferenceOffset));
+    }
+
+    public ByteBuffer readBytes(int recordReferenceOffset, int size) {
+        return internalReadBytes(index(recordReferenceOffset), size);
+    }
+
+    private ByteBuffer internalReadBytes(int index, int size) {
+        ByteBuffer duplicate = buffer.duplicate();
+        duplicate.position(index);
+        duplicate.limit(index + size);
+        return duplicate.slice();
+    }
+
+    public int size() {
+        return buffer.remaining();
+    }
+
+    public void hexDump(OutputStream stream) throws IOException {
+        byte[] data = new byte[buffer.remaining()];
+        buffer.duplicate().get(data);
+        HexDump.dump(data, 0, stream, 0);
+    }
+
+    public void binDump(OutputStream stream) throws IOException {
+        ByteBuffer data = buffer.duplicate();
+        try (WritableByteChannel channel = Channels.newChannel(stream)) {
+            while (data.hasRemaining()) {
+                channel.write(data);
+            }
+        }
+    }
+
+    public int estimateMemoryUsage() {
+        return buffer.isDirect() ? 0 : buffer.remaining();
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/SegmentData.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java?rev=1804675&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java
 Thu Aug 10 15:00:43 2017
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.data;
+
+public class StringData {
+
+    private final String string;
+
+    private final RecordIdData recordId;
+
+    private final int length;
+
+    StringData(String string, int length) {
+        this.string = string;
+        this.length = length;
+        this.recordId = null;
+    }
+
+    StringData(RecordIdData recordId, int length) {
+        this.recordId = recordId;
+        this.length = length;
+        this.string = null;
+    }
+
+    public boolean isString() {
+        return string != null;
+    }
+
+    public boolean isRecordId() {
+        return recordId != null;
+    }
+
+    public String getString() {
+        return string;
+    }
+
+    public RecordIdData getRecordId() {
+         return recordId;
+    }
+
+    public int getLength() {
+        return length;
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/data/StringData.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1804675&r1=1804674&r2=1804675&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
 Thu Aug 10 15:00:43 2017
@@ -18,6 +18,8 @@
  */
 package org.apache.jackrabbit.oak.segment.file;
 
+import static 
org.apache.jackrabbit.oak.segment.data.SegmentData.newSegmentData;
+
 import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
@@ -207,7 +209,7 @@ public abstract class AbstractFileStore
         long msb = id.getMostSignificantBits();
         long lsb = id.getLeastSignificantBits();
         ByteBuffer buffer = ByteBuffer.wrap(data);
-        GCGeneration generation = Segment.getGcGeneration(buffer, id);
+        GCGeneration generation = 
Segment.getGcGeneration(newSegmentData(buffer), id);
         w.recoverEntry(msb, lsb, data, 0, data.length, generation);
         if (SegmentId.isDataSegmentId(lsb)) {
             Segment segment = new Segment(tracker, segmentReader, 
tracker.newSegmentId(msb, lsb), buffer);


Reply via email to