Author: frm
Date: Thu Aug 3 12:56:07 2017
New Revision: 1803995
URL: http://svn.apache.org/viewvc?rev=1803995&view=rev
Log:
OAK-6518 - Extract index load and validation logic in different class and
package
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
(with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java?rev=1803995&r1=1803994&r2=1803995&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
Thu Aug 3 12:56:07 2017
@@ -33,7 +33,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BINARY_REFERENCES_MAGIC;
import static
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
import static
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC;
-import static
org.apache.jackrabbit.oak.segment.file.tar.TarConstants.INDEX_MAGIC;
+import static
org.apache.jackrabbit.oak.segment.file.tar.index.IndexLoader.newIndexLoader;
import java.io.Closeable;
import java.io.File;
@@ -62,6 +62,8 @@ import com.google.common.base.Predicate;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.segment.file.tar.index.IndexLoader;
+import org.apache.jackrabbit.oak.segment.file.tar.index.InvalidIndexException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,6 +71,8 @@ class TarReader implements Closeable {
private static final Logger log = LoggerFactory.getLogger(TarReader.class);
+ private static final IndexLoader indexLoader = newIndexLoader(BLOCK_SIZE);
+
/**
* Pattern of the segment entry names. Note the trailing (\\..*)? group
* that's included for compatibility with possible future extensions.
@@ -335,79 +339,12 @@ class TarReader implements Closeable {
* instead.
*/
private static ByteBuffer loadAndValidateIndex(RandomAccessFile file,
String name) throws IOException {
- long length = file.length();
- if (length % BLOCK_SIZE != 0
- || length < 6 * BLOCK_SIZE
- || length > Integer.MAX_VALUE) {
- log.warn("Unexpected size {} of tar file {}", length, name);
- return null; // unexpected file size
- }
-
- // read the index metadata just before the two final zero blocks
- ByteBuffer meta = ByteBuffer.allocate(16);
- file.seek(length - 2 * BLOCK_SIZE - 16);
- file.readFully(meta.array());
- int crc32 = meta.getInt();
- int count = meta.getInt();
- int bytes = meta.getInt();
- int magic = meta.getInt();
-
- if (magic != INDEX_MAGIC) {
- return null; // magic byte mismatch
- }
-
- if (count < 1 || bytes < count * TarEntry.SIZE + 16 || bytes %
BLOCK_SIZE != 0) {
- log.warn("Invalid index metadata in tar file {}", name);
- return null; // impossible entry and/or byte counts
- }
-
- // this involves seeking backwards in the file, which might not
- // perform well, but that's OK since we only do this once per file
- ByteBuffer index = ByteBuffer.allocate(count * TarEntry.SIZE);
- file.seek(length - 2 * BLOCK_SIZE - 16 - count * TarEntry.SIZE);
- file.readFully(index.array());
- index.mark();
-
- CRC32 checksum = new CRC32();
- long limit = length - 2 * BLOCK_SIZE - bytes - BLOCK_SIZE;
- long lastmsb = Long.MIN_VALUE;
- long lastlsb = Long.MIN_VALUE;
- byte[] entry = new byte[TarEntry.SIZE];
- for (int i = 0; i < count; i++) {
- index.get(entry);
- checksum.update(entry);
-
- ByteBuffer buffer = wrap(entry);
- long msb = buffer.getLong();
- long lsb = buffer.getLong();
- int offset = buffer.getInt();
- int size = buffer.getInt();
-
- if (lastmsb > msb || (lastmsb == msb && lastlsb > lsb)) {
- log.warn("Incorrect index ordering in tar file {}", name);
- return null;
- } else if (lastmsb == msb && lastlsb == lsb && i > 0) {
- log.warn("Duplicate index entry in tar file {}", name);
- return null;
- } else if (offset < 0 || offset % BLOCK_SIZE != 0) {
- log.warn("Invalid index entry offset in tar file {}", name);
- return null;
- } else if (size < 1 || offset + size > limit) {
- log.warn("Invalid index entry size in tar file {}", name);
- return null;
- }
-
- lastmsb = msb;
- lastlsb = lsb;
- }
-
- if (crc32 != (int) checksum.getValue()) {
- log.warn("Invalid index checksum in tar file {}", name);
- return null; // checksum mismatch
+ try {
+ return indexLoader.loadIndex(file);
+ } catch (InvalidIndexException e) {
+ log.warn("Unable to load index of file {}: {}", name,
e.getMessage());
}
-
- index.reset();
- return index;
+ return null;
}
/**
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java?rev=1803995&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
Thu Aug 3 12:56:07 2017
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.nio.ByteBuffer.wrap;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
+public class IndexLoader {
+
+ private static final int MAGIC = ('\n' << 24) + ('0' << 16) + ('K' << 8) +
'\n';
+
+ private static final int ENTRY_SIZE = 33;
+
+ public static IndexLoader newIndexLoader(int blockSize) {
+ checkArgument(blockSize > 0, "Invalid block size");
+ return new IndexLoader(blockSize);
+ }
+
+ private final int blockSize;
+
+ private IndexLoader(int blockSize) {
+ this.blockSize = blockSize;
+ }
+
+ public ByteBuffer loadIndex(RandomAccessFile file) throws IOException,
InvalidIndexException {
+ long length = file.length();
+
+ if (length % blockSize != 0 || length < 6 * blockSize || length >
Integer.MAX_VALUE) {
+ throw new InvalidIndexException(String.format("Unexpected size
%d", length));
+ }
+
+ // read the index metadata just before the two final zero blocks
+ ByteBuffer meta = ByteBuffer.allocate(16);
+ file.seek(length - 2 * blockSize - 16);
+ file.readFully(meta.array());
+ int crc32 = meta.getInt();
+ int count = meta.getInt();
+ int bytes = meta.getInt();
+ int magic = meta.getInt();
+
+ if (magic != MAGIC) {
+ return null; // magic byte mismatch
+ }
+
+ if (count < 1 || bytes < count * ENTRY_SIZE + 16 || bytes % blockSize
!= 0) {
+ throw new InvalidIndexException("Invalid metadata");
+ }
+
+ // this involves seeking backwards in the file, which might not
+ // perform well, but that's OK since we only do this once per file
+ ByteBuffer index = ByteBuffer.allocate(count * ENTRY_SIZE);
+ file.seek(length - 2 * blockSize - 16 - count * ENTRY_SIZE);
+ file.readFully(index.array());
+ index.mark();
+
+ CRC32 checksum = new CRC32();
+ long limit = length - 2 * blockSize - bytes - blockSize;
+ long lastMsb = Long.MIN_VALUE;
+ long lastLsb = Long.MIN_VALUE;
+ byte[] entry = new byte[ENTRY_SIZE];
+ for (int i = 0; i < count; i++) {
+ index.get(entry);
+ checksum.update(entry);
+
+ ByteBuffer buffer = wrap(entry);
+ long msb = buffer.getLong();
+ long lsb = buffer.getLong();
+ int offset = buffer.getInt();
+ int size = buffer.getInt();
+
+ if (lastMsb > msb || (lastMsb == msb && lastLsb > lsb)) {
+ throw new InvalidIndexException("Incorrect entry ordering");
+ }
+ if (lastMsb == msb && lastLsb == lsb && i > 0) {
+ throw new InvalidIndexException("Duplicate entry");
+ }
+ if (offset < 0 || offset % blockSize != 0) {
+ throw new InvalidIndexException("Invalid entry offset");
+ }
+ if (size < 1 || offset + size > limit) {
+ throw new InvalidIndexException("Invalid entry size");
+ }
+
+ lastMsb = msb;
+ lastLsb = lsb;
+ }
+
+ if (crc32 != (int) checksum.getValue()) {
+ throw new InvalidIndexException("Invalid checksum");
+ }
+
+ index.reset();
+ return index;
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java?rev=1803995&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java
Thu Aug 3 12:56:07 2017
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+public class InvalidIndexException extends Exception {
+
+ InvalidIndexException(String message) {
+ super(message);
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/InvalidIndexException.java
------------------------------------------------------------------------------
svn:eol-style = native
