Author: amitj
Date: Wed Aug 30 06:40:06 2017
New Revision: 1806668
URL: http://svn.apache.org/viewvc?rev=1806668&view=rev
Log:
OAK-6596: Blob store consistency check can show bogus errors about missing blobs
- Patch from Csaba Varga
Modified:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
Modified:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java?rev=1806668&r1=1806667&r2=1806668&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/FileIOUtils.java
Wed Aug 30 06:40:06 2017
@@ -95,7 +95,7 @@ public final class FileIOUtils {
*/
public static void sort(File file, Comparator<String> comparator) throws
IOException {
File sorted = createTempFile("fleioutilssort", null);
- merge(sortInBatch(file, comparator, true), sorted);
+ merge(sortInBatch(file, comparator, true), sorted, comparator);
move(sorted, file);
}
@@ -113,6 +113,19 @@ public final class FileIOUtils {
}
/**
+ * Merges a list of files after sorting with the given comparator.
+ *
+ * @param files files to merge
+ * @param output merge output file
+ * @throws IOException
+ */
+ public static void merge(List<File> files, File output, Comparator<String>
comparator) throws IOException {
+ mergeSortedFiles(
+ files,
+ output, comparator, true);
+ }
+
+ /**
* Copies an input stream to a file.
*
Modified:
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java?rev=1806668&r1=1806667&r2=1806668&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
(original)
+++
jackrabbit/oak/trunk/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FileIOUtilsTest.java
Wed Aug 30 06:40:06 2017
@@ -40,7 +40,9 @@ import java.util.Set;
import javax.annotation.Nullable;
import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Sets;
+import com.google.common.primitives.Longs;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.FileIOUtils.BurnOnCloseFileIterator;
import org.junit.Assert;
@@ -65,6 +67,7 @@ import static org.apache.jackrabbit.oak.
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
/**
@@ -160,6 +163,46 @@ public class FileIOUtilsTest {
Collections.sort(list);
assertArrayEquals(Arrays.toString(list.toArray()), list.toArray(),
retrieved.toArray());
}
+
+ @Test
+ public void sortLargeFileWithCustomComparatorTest() throws IOException {
+ final int numEntries = 100000; // must be large enough to trigger
split/merge functionality of the sort
+ long[] entries = new long[numEntries];
+ Random r = new Random(0);
+ for (int i = 0; i < numEntries; i++) {
+ entries[i] = r.nextLong();
+ }
+
+ Iterator<Long> boxedEntries = Longs.asList(entries).iterator();
+ Iterator<String> hexEntries = Iterators.transform(boxedEntries, new
Function<Long, String>() {
+ @Nullable @Override public String apply(@Nullable Long
input) {
+ return Long.toHexString(input);
+ }
+ });
+ File f = assertWrite(hexEntries, false, numEntries);
+
+ Comparator<String> prefixComparator = new Comparator<String>() {
+ @Override public int compare(String s1, String s2) {
+ return s1.substring(0, 3).compareTo(s2.substring(0, 3));
+ }
+ };
+
+ sort(f, prefixComparator);
+ BufferedReader reader =
+ new BufferedReader(new InputStreamReader(new FileInputStream(f),
UTF_8));
+ String previous = reader.readLine().substring(0, 3);
+ while (true) {
+ String current = reader.readLine();
+ if (current == null) {
+ break;
+ }
+ current = current.substring(0, 3);
+ assertFalse("Distinct sort didn't filter out duplicates
properly.", previous.equals(current));
+ assertTrue("Sort didn't create increasing order",
previous.compareTo(current) < 0);
+ previous = current;
+ }
+ closeQuietly(reader);
+ }
@Test
public void testCopy() throws IOException{
