Author: amitj
Date: Wed Jan 27 08:21:41 2016
New Revision: 1726952
URL: http://svn.apache.org/viewvc?rev=1726952&view=rev
Log:
OAK-3184: Consistency checker for data/blob store
Merged revision(s) 1702371, 1702426, 1702428 from trunk
Added:
jackrabbit/oak/branches/1.2/oak-commons/src/test/resources/org/apache/jackrabbit/oak/commons/sort/test-file-2.csv
- copied unchanged from r1702426,
jackrabbit/oak/trunk/oak-commons/src/test/resources/org/apache/jackrabbit/oak/commons/sort/test-file-2.csv
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java
- copied unchanged from r1702371,
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
jackrabbit/oak/branches/1.2/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/sort/ExternalSortTest.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan 27 08:21:41 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618
,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695280,1695299,1695420,1695457,1695482,1695492,1695
507,1695521,1695540,1695571,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697410,1697582,1697589,1697616,1697672,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702387,1702405,1702423,1702860,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1706764,1706772,1707049,1707191,1707331,1707435,1707509,1708105,1708315,1708546,1708592,1708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1710575,1710590,1710614,1
710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715767,1715771,1715888,1715898,1716178,1716426,1716576,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717277,1717462,1717632,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720335,1720350,1720354,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,1723350,1723584,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725555,1725960,1726232,1726570,1726579,1726585-1726586,1726795,1726797,1726809,172681
2
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681921,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618
,1684669,1684820,1684868,1684894,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686003,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693050,1693209,1693401,1693421,1693525-1693526,1694007,1694393-1694394,1694651,1694653-1694654,1695032,1695050,1695122,1695280,1695299,1695420,1695457,1695482,1695492,1695
507,1695521,1695540,1695571,1695905,1696190,1696194,1696242,1696285,1696375,1696522,1696578,1696759,1696916,1697363,1697373,1697410,1697582,1697589,1697616,1697672,1700191,1700231,1700397,1700403,1700506,1700571,1700718,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701907,1701948,1701955,1701959,1701965,1701986,1702014,1702022,1702045,1702051,1702241,1702272,1702371,1702387,1702405,1702423,1702426,1702428,1702860,1702942,1702960,1703212,1703382,1703395,1703411,1703428,1703430,1703568,1703592,1703758,1703858,1703878,1704256,1704282,1704285,1704457,1704479,1704490,1704614,1704629,1704636,1704655,1704670,1704886,1705005,1705027,1705043,1705055,1705250,1705268,1705273,1705323,1705677,1705701,1705871,1705992,1705998,1706009,1706037,1706059,1706212,1706218,1706270,1706764,1706772,1707049,1707191,1707331,1707435,1707509,1708105,1708315,1708546,1708592,1708766,1709012,1709852,1709978,1710013,1710031,1710049,1710205,1710242,1710559,1
710575,1710590,1710614,1710637,1710789,1710800,1710811,1710816,1710972,1711248,1711282,1711296,1711405,1711498,1711654,1712018,1712042,1712319,1712490,1712531,1712730,1712785,1712963,1713008,1713439,1713461,1713580,1713586,1713599-1713600,1713626,1713698,1713803,1713809,1714034,1714061,1714084,1714170,1714213,1714229,1714238,1714519-1714520,1714543-1714544,1714730,1714739,1714779,1714956,1714961,1715010,1715092,1715191,1715346,1715767,1715771,1715888,1715898,1716178,1716426,1716576,1716596,1716616,1716703,1716712,1716815,1716823,1716830,1716883,1717277,1717462,1717632,1717784,1717789,1717988,1718528,1718533,1718547-1718548,1718626,1718646,1718772,1718801-1718802,1718895,1719111,1719288,1719869,1720335,1720350,1720354,1720500,1721160,1721172,1721337,1722141,1722832,1723227,1723239,1723241,1723251,1723254,1723333,1723347,1723350,1723584,1723713,1723731,1724026,1724057,1724186,1724210,1724401,1724628,1724631,1725216,1725477,1725555,1725960,1726232,1726570,1726579,1726585-1726586,172679
5,1726797,1726809,1726812
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.2/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
Wed Jan 27 08:21:41 2016
@@ -295,7 +295,7 @@ public class ExternalSort {
try {
for (String r : tmplist) {
// Skip duplicate lines
- if (!distinct || !r.equals(lastLine)) {
+ if (!distinct || (lastLine == null || (lastLine != null &&
cmp.compare(r, lastLine) != 0))) {
writeLine(fbw, r);
fbw.newLine();
lastLine = r;
@@ -453,7 +453,7 @@ public class ExternalSort {
BinaryFileBuffer bfb = pq.poll();
String r = bfb.pop();
// Skip duplicate lines
- if (!distinct || !r.equals(lastLine)) {
+ if (!distinct || (lastLine == null || (lastLine != null &&
cmp.compare(r, lastLine) != 0))) {
writeLine(fbw, r);
fbw.newLine();
lastLine = r;
Modified:
jackrabbit/oak/branches/1.2/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/sort/ExternalSortTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/sort/ExternalSortTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/sort/ExternalSortTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/sort/ExternalSortTest.java
Wed Jan 27 08:21:41 2016
@@ -78,6 +78,8 @@ public class ExternalSortTest {
private static final String TEST_FILE1_TXT = "test-file-1.txt";
private static final String TEST_FILE2_TXT = "test-file-2.txt";
private static final String TEST_FILE1_CSV = "test-file-1.csv";
+ private static final String TEST_FILE2_CSV = "test-file-2.csv";
+
private static final String[] EXPECTED_SORT_RESULTS = { "a", "b", "b", "e",
"f", "i", "m", "o", "u", "u", "x", "y", "z" };
private static final String[] EXPECTED_MERGE_RESULTS = { "a", "a", "b",
@@ -90,10 +92,14 @@ public class ExternalSortTest {
"f", "i", "m", "o", "u", "x", "y", "z" };
private static final String[] SAMPLE = { "f", "m", "b", "e", "i", "o", "u",
"x", "a", "y", "z", "b", "u" };
-
+ private static final String[] EXPECTED_CSV_DISTINCT_RESULTS = { "a,1",
"b,2a", "e,3", "f,4", "i,5", "m,6", "o,7",
+ "u,8a",
"x,9", "y,10", "z,11"};
+ private static final String[] EXPECTED_CSV_RESULTS = { "a,1", "b,2a",
"b,2b", "e,3", "f,4", "i,5", "m,6", "o,7",
+ "u,8a", "u,8b",
"x,9", "y,10", "z,11"};
private File file1;
private File file2;
private File csvFile;
+ private File csvFile2;
private List<File> fileList;
/**
@@ -108,6 +114,8 @@ public class ExternalSortTest {
getResource(TEST_FILE2_TXT).toURI());
this.csvFile = new File(this.getClass().
getResource(TEST_FILE1_CSV).toURI());
+ this.csvFile2 = new File(this.getClass().
+ getResource(TEST_FILE2_CSV).toURI());
File tmpFile1 = new File(this.file1.getPath().toString() + ".tmp");
File tmpFile2 = new File(this.file2.getPath().toString() + ".tmp");
@@ -340,6 +348,51 @@ public class ExternalSortTest {
testCSVSortingWithParams(true);
}
+ @Test
+ public void testCSVKeyValueSorting() throws Exception {
+ testCSVSortKeyValue(false);
+ testCSVSortKeyValue(true);
+ }
+
+ /**
+ * Sample case to sort csv file with key, value pair.
+ *
+ * @param distinct if distinct records need to be omitted
+ * @throws Exception
+ *
+ */
+ public void testCSVSortKeyValue(boolean distinct) throws Exception {
+
+ File out = File.createTempFile("test_results", ".tmp", null);
+
+ Comparator<String> cmp = new Comparator<String>() {
+ @Override
+ public int compare(String s1, String s2) {
+ return s1.split(",")[0].compareTo(s2.split(",")[0]);
+ }
+ };
+
+ List<File> listOfFiles = ExternalSort.sortInBatch(this.csvFile2, cmp,
+
ExternalSort.DEFAULTMAXTEMPFILES,
+
ExternalSort.DEFAULT_MAX_MEM_BYTES,
+
Charset.defaultCharset(),
+ null, distinct,
0, false);
+
+ // now merge with append
+ ExternalSort.mergeSortedFiles(listOfFiles, out, cmp,
+ Charset.defaultCharset(), distinct,
true, false);
+ ArrayList<String> result = readLines(out);
+
+ if (distinct) {
+ assertEquals(11, result.size());
+ assertArrayEquals(Arrays.toString(result.toArray()),
EXPECTED_CSV_DISTINCT_RESULTS, result.toArray());
+ } else {
+ assertEquals(13, result.size());
+ assertArrayEquals(Arrays.toString(result.toArray()),
EXPECTED_CSV_RESULTS, result.toArray());
+ }
+
+ }
+
/**
* Sample case to sort csv file.
*
@@ -404,4 +457,4 @@ public class ExternalSortTest {
}
}
-}
\ No newline at end of file
+}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
Wed Jan 27 08:21:41 2016
@@ -59,7 +59,11 @@ public class BlobGC extends AnnotatedSt
private final Executor executor;
private ManagementOperation<String> gcOp = done(OP_NAME, "");
-
+
+ public static final String CONSISTENCY_OP_NAME = "Blob consistency check";
+
+ private ManagementOperation<String> consistencyOp =
done(CONSISTENCY_OP_NAME, "");
+
/**
* @param blobGarbageCollector Blob garbage collector
* @param executor executor for running the garbage
collection task
@@ -114,6 +118,30 @@ public class BlobGC extends AnnotatedSt
return tds;
}
+ @Override
+ public CompositeData checkConsistency() {
+ if (consistencyOp.isDone()) {
+ consistencyOp = newManagementOperation(CONSISTENCY_OP_NAME, new
Callable<String>() {
+ @Override
+ public String call() throws Exception {
+ long t0 = nanoTime();
+ long missing = blobGarbageCollector.checkConsistency();
+ return "Consistency check completed in "
+ + formatTime(nanoTime() - t0) + " " +
+ + missing + "missing blobs found (details in
the log).";
+ }
+ });
+ executor.execute(consistencyOp);
+ }
+ return getConsistencyCheckStatus();
+ }
+
+ @Nonnull
+ @Override
+ public CompositeData getConsistencyCheckStatus() {
+ return consistencyOp.getStatus().toCompositeData();
+ }
+
private CompositeDataSupport toCompositeData(GarbageCollectionRepoStats
statObj) throws OpenDataException {
Object[] values = new Object[] {
statObj.getRepositoryId(),
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
Wed Jan 27 08:21:41 2016
@@ -61,5 +61,20 @@ public interface BlobGCMBean {
* @return List of available repositories and their status
*/
TabularData getGlobalMarkStats();
-
+
+ /**
+ * Data Store consistency check
+ *
+ * @return the missing blobs
+ */
+ CompositeData checkConsistency();
+
+ /**
+ * Consistency check status
+ *
+ * @return the status of the ongoing operation or if none the terminal
+ * status of the last operation or <em>Status not available</em> if none.
+ */
+ @Nonnull
+ CompositeData getConsistencyCheckStatus();
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
Wed Jan 27 08:21:41 2016
@@ -39,4 +39,12 @@ public interface BlobGarbageCollector {
* @throws Exception
*/
List<GarbageCollectionRepoStats> getStats() throws Exception;
+
+ /**
+ * Checks for consistency in the blob store and reporting the number of
missing blobs.
+ *
+ * @return number of inconsistencies
+ * @throws Exception
+ */
+ long checkConsistency() throws Exception;
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
Wed Jan 27 08:21:41 2016
@@ -134,6 +134,20 @@ public class GarbageCollectorFileState i
merge(ExternalSort.sortInBatch(file, lexComparator, true), sorted);
Files.move(sorted, file);
}
+
+ /**
+ * Sorts the given file externally with the given comparator.
+ *
+ * @param file file whose contents needs to be sorted
+ * @param comparator to compare
+ * @throws IOException
+ */
+ public static void sort(File file, Comparator<String> comparator) throws
IOException {
+ File sorted = createTempFile();
+ merge(ExternalSort.sortInBatch(file, comparator, true), sorted);
+ Files.move(sorted, file);
+ }
+
public static void merge(List<File> files, File output) throws IOException
{
ExternalSort.mergeSortedFiles(
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Wed Jan 27 08:21:41 2016
@@ -27,12 +27,14 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.sql.Timestamp;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -49,11 +51,13 @@ import com.google.common.collect.Maps;
import com.google.common.collect.PeekingIterator;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import com.google.common.util.concurrent.ListenableFutureTask;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -80,7 +84,9 @@ public class MarkSweepGarbageCollector i
public static final String TEMP_DIR =
StandardSystemProperty.JAVA_IO_TMPDIR.value();
public static final int DEFAULT_BATCH_COUNT = 2048;
-
+
+ public static final String DELIM = ",";
+
/** The last modified time before current time of blobs to consider for
garbage collection. */
private final long maxLastModifiedInterval;
@@ -286,13 +292,18 @@ public class MarkSweepGarbageCollector i
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getMarkedRefs(),
fs.getAvailableRefs());
+ calculateDifference(fs, iter);
+ LOG.debug("Ending difference phase of the garbage collector");
+ }
+
+ private long calculateDifference(GarbageCollectorFileState fs,
FileLineDifferenceIterator iter) throws IOException {
+ long numCandidates = 0;
BufferedWriter bufferWriter = null;
try {
bufferWriter = Files.newWriter(fs.getGcCandidates(),
Charsets.UTF_8);
List<String> expiredSet = newArrayList();
- int numCandidates = 0;
while (iter.hasNext()) {
expiredSet.add(iter.next());
if (expiredSet.size() > getBatchCount()) {
@@ -305,15 +316,14 @@ public class MarkSweepGarbageCollector i
numCandidates += expiredSet.size();
saveBatchToFile(expiredSet, bufferWriter);
}
- LOG.debug("Found GC candidates - " + numCandidates);
+ LOG.debug("Found candidates - " + numCandidates);
} finally {
IOUtils.closeQuietly(bufferWriter);
IOUtils.closeQuietly(iter);
}
-
- LOG.debug("Ending difference phase of the garbage collector");
+ return numCandidates;
}
-
+
/**
* Sweep phase of gc candidate deletion.
* <p>
@@ -480,16 +490,18 @@ public class MarkSweepGarbageCollector i
private final boolean debugMode = LOG.isTraceEnabled();
@Override
- public void addReference(String blobId) {
+ public void addReference(String blobId, String nodeId)
{
if (debugMode) {
- LOG.trace("BlobId : {}", blobId);
+ LOG.trace("BlobId : {}, NodeId : {}", blobId,
nodeId);
}
try {
Iterator<String> idIter =
blobStore.resolveChunks(blobId);
+ Joiner delimJoiner =
Joiner.on(DELIM).skipNulls();
while (idIter.hasNext()) {
String id = idIter.next();
- idBatch.add(id);
+
+ idBatch.add(delimJoiner.join(id, nodeId));
if (idBatch.size() >= getBatchCount()) {
saveBatchToFile(idBatch, writer);
@@ -518,14 +530,67 @@ public class MarkSweepGarbageCollector i
);
LOG.info("Number of valid blob references marked under mark phase
of " +
"Blob garbage collection [{}]", count.get());
- // sort the marked references
- GarbageCollectorFileState.sort(fs.getMarkedRefs());
+ // sort the marked references with the first part of the key
+ GarbageCollectorFileState.sort(fs.getMarkedRefs(),
+ new Comparator<String>() {
+ @Override
+ public int compare(String
s1, String s2) {
+ return
s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
+ }
+ });
} finally {
IOUtils.closeQuietly(writer);
}
}
-
-
+
+ /**
+ * Checks for the DataStore consistency and reports the number of missing
blobs still referenced.
+ *
+ * @return the missing blobs
+ * @throws Exception
+ */
+ @Override
+ public long checkConsistency() throws Exception {
+ boolean threw = true;
+ GarbageCollectorFileState fs = new GarbageCollectorFileState(root);
+ long candidates = 0;
+
+ try {
+ Stopwatch sw = Stopwatch.createStarted();
+ LOG.info("Starting blob consistency check");
+
+ // Find all blobs available in the blob store
+ ListenableFutureTask<Integer> blobIdRetriever =
ListenableFutureTask.create(new BlobIdRetriever(fs));
+ executor.execute(blobIdRetriever);
+
+ // Mark all used blob references
+ iterateNodeTree(fs);
+
+ try {
+ blobIdRetriever.get();
+ } catch (ExecutionException e) {
+ LOG.warn("Error occurred while fetching all the blobIds from
the BlobStore");
+ threw = false;
+ throw e;
+ }
+
+ LOG.trace("Starting difference phase of the consistency check");
+ FileLineDifferenceIterator iter = new
FileLineDifferenceIterator(fs.getAvailableRefs(), fs.getMarkedRefs());
+ candidates = calculateDifference(fs, iter);
+ LOG.trace("Ending difference phase of the consistency check");
+
+ LOG.info("Consistency check found [{}] missing blobs", candidates);
+ if (candidates > 0) {
+ LOG.warn("Consistency check failure in the the blob store :
{}, check missing candidates in file {}",
+ blobStore, fs.getGcCandidates().getAbsolutePath());
+ }
+ } finally {
+ if (!LOG.isTraceEnabled() && candidates == 0) {
+ Closeables.close(fs, threw);
+ }
+ }
+ return candidates;
+ }
/**
* BlobIdRetriever class to retrieve all blob ids.
*/
@@ -564,7 +629,7 @@ public class MarkSweepGarbageCollector i
// sort the file
GarbageCollectorFileState.sort(fs.getAvailableRefs());
- LOG.debug("Number of blobs present in BlobStore : [{}] ",
blobsCount);
+ LOG.info("Number of blobs present in BlobStore : [{}] ",
blobsCount);
} finally {
IOUtils.closeQuietly(bufferWriter);
}
@@ -606,7 +671,11 @@ public class MarkSweepGarbageCollector i
LineIterator.closeQuietly(marked);
LineIterator.closeQuietly(all);
}
-
+
+ private String getKey(String row) {
+ return row.split(DELIM)[0];
+ }
+
private String computeNextDiff() {
if (!all.hasNext()) {
return null;
@@ -622,7 +691,7 @@ public class MarkSweepGarbageCollector i
diff = all.next();
while (peekMarked.hasNext()) {
String marked = peekMarked.peek();
- int comparisonResult = diff.compareTo(marked);
+ int comparisonResult =
getKey(diff).compareTo(getKey(marked));
if (comparisonResult > 0) {
//Extra entries in marked. Ignore them and move on
peekMarked.next();
@@ -635,7 +704,12 @@ public class MarkSweepGarbageCollector i
} else {
//This entry is not found in marked entries
//hence part of diff
- return diff;
+ if (!InMemoryDataRecord.isInstance(getKey(diff))) {
+ return diff;
+ } else {
+ diff = null;
+ break;
+ }
}
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
Wed Jan 27 08:21:41 2016
@@ -18,14 +18,21 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import javax.annotation.Nullable;
+
/**
* Callback interface for collecting all blob references that are
* potentially accessible. Useful for marking referenced blobs as
* in use when collecting garbage in an external data store.
*/
public interface ReferenceCollector {
-
- void addReference(String reference);
-
+
+ /**
+ * Adds the reference detected with the node Id.
+ *
+ * @param reference
+ * @param nodeId
+ */
+ void addReference(String reference, @Nullable String nodeId);
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
Wed Jan 27 08:21:41 2016
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
/**
* Represents binary data which is backed by a byte[] (in memory).
*/
-class InMemoryDataRecord implements DataRecord {
+public class InMemoryDataRecord implements DataRecord {
/**
* Logger instance for this class
@@ -89,7 +89,7 @@ class InMemoryDataRecord implements Data
* @param id DataRecord identifier
* @return true if it can be converted
*/
- static boolean isInstance(String id) {
+ public static boolean isInstance(String id) {
return id.startsWith(PREFIX);
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
Wed Jan 27 08:21:41 2016
@@ -27,6 +27,7 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.json.JsopReader;
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
public class BlobCollector {
@@ -36,7 +37,7 @@ public class BlobCollector {
this.nodeStore = nodeStore;
}
- public void collect(NodeDocument doc, Collection<Blob> blobs) {
+ public void collect(NodeDocument doc, Collection<ReferencedBlob> blobs) {
for (String key : doc.keySet()) {
if (!Utils.isPropertyName(key)) {
continue;
@@ -44,13 +45,13 @@ public class BlobCollector {
Map<Revision, String> valueMap = doc.getLocalMap(key);
for (String v : valueMap.values()) {
if (v != null) {
- loadValue(v, blobs);
+ loadValue(v, blobs, doc.getPath());
}
}
}
}
- private void loadValue(String v, Collection<Blob> blobs) {
+ private void loadValue(String v, Collection<ReferencedBlob> blobs, String
nodeId) {
JsopReader reader = new JsopTokenizer(v);
PropertyState p;
if (reader.matches('[')) {
@@ -58,14 +59,14 @@ public class BlobCollector {
if (p.getType() == Type.BINARIES) {
for (int i = 0; i < p.count(); i++) {
Blob b = p.getValue(Type.BINARY, i);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, nodeId));
}
}
} else {
p = DocumentPropertyState.readProperty("x", nodeStore, reader);
if (p.getType() == Type.BINARY) {
Blob b = p.getValue(Type.BINARY);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, nodeId));
}
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
Wed Jan 27 08:21:41 2016
@@ -21,7 +21,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
-import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
/**
* An iterator over all referenced binaries.
@@ -30,13 +30,13 @@ import org.apache.jackrabbit.oak.api.Blo
* The items are returned in no particular order.
* An item might be returned multiple times.
*/
-public class BlobReferenceIterator implements Iterator<Blob> {
+public class BlobReferenceIterator implements Iterator<ReferencedBlob> {
private static final int BATCH_SIZE = 1000;
private final DocumentStore docStore;
private final BlobCollector blobCollector;
- private HashSet<Blob> batch = new HashSet<Blob>();
- private Iterator<Blob> batchIterator;
+ private HashSet<ReferencedBlob> batch = new HashSet<ReferencedBlob>();
+ private Iterator<ReferencedBlob> batchIterator;
private boolean done;
private String fromKey = NodeDocument.MIN_ID_VALUE;
@@ -55,7 +55,7 @@ public class BlobReferenceIterator imple
}
@Override
- public Blob next() {
+ public ReferencedBlob next() {
// this will load the next batch if required
if (!hasNext()) {
throw new NoSuchElementException();
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
Wed Jan 27 08:21:41 2016
@@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,23 +44,24 @@ public class DocumentBlobReferenceRetrie
@Override
public void collectReferences(ReferenceCollector collector) {
int referencesFound = 0;
- Iterator<Blob> blobIterator = nodeStore.getReferencedBlobsIterator();
+ Iterator<ReferencedBlob> blobIterator =
nodeStore.getReferencedBlobsIterator();
try {
while (blobIterator.hasNext()) {
- Blob blob = blobIterator.next();
+ ReferencedBlob refBlob = blobIterator.next();
+ Blob blob = refBlob.getBlob();
referencesFound++;
//TODO this mode would also add in memory blobId
//Would that be an issue
if (blob instanceof BlobStoreBlob) {
- collector.addReference(((BlobStoreBlob) blob).getBlobId());
+ collector.addReference(((BlobStoreBlob) blob).getBlobId(),
refBlob.getId());
} else {
//TODO Should not rely on toString. Instead obtain
//secure reference and convert that to blobId using
//blobStore
- collector.addReference(blob.toString());
+ collector.addReference(blob.toString(), refBlob.getId());
}
}
}finally{
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
Wed Jan 27 08:21:41 2016
@@ -82,6 +82,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.Checkpoints.Info;
import org.apache.jackrabbit.oak.plugins.document.cache.CacheInvalidationStats;
import
org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobReferenceIterator;
@@ -2608,7 +2609,7 @@ public final class DocumentNodeStore
* @see
org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobReferenceIterator
* @return an iterator for all the blobs
*/
- public Iterator<Blob> getReferencedBlobsIterator() {
+ public Iterator<ReferencedBlob> getReferencedBlobsIterator() {
if(store instanceof MongoDocumentStore){
return new MongoBlobReferenceIterator(this, (MongoDocumentStore)
store);
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
Wed Jan 27 08:21:41 2016
@@ -28,16 +28,16 @@ import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.QueryBuilder;
-import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.BlobCollector;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
-public class MongoBlobReferenceIterator extends AbstractIterator<Blob>
implements Closeable {
+public class MongoBlobReferenceIterator extends
AbstractIterator<ReferencedBlob> implements Closeable {
private final MongoDocumentStore documentStore;
private final BlobCollector blobCollector;
- private final Queue<Blob> blobs = Queues.newArrayDeque();
+ private final Queue<ReferencedBlob> blobs = Queues.newArrayDeque();
private DBCursor cursor;
@@ -48,7 +48,7 @@ public class MongoBlobReferenceIterator
}
@Override
- protected Blob computeNext() {
+ protected ReferencedBlob computeNext() {
if (blobs.isEmpty()) {
loadBatch();
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
Wed Jan 27 08:21:41 2016
@@ -305,7 +305,7 @@ public class Segment {
for (int i = 0; i < blobrefcount; i++) {
int offset = (data.getShort(blobrefpos + i * 2) & 0xffff) << 2;
SegmentBlob blob = new SegmentBlob(new RecordId(id, offset));
- collector.addReference(blob.getBlobId());
+ collector.addReference(blob.getBlobId(), null);
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
Wed Jan 27 08:21:41 2016
@@ -89,7 +89,21 @@ public class FileLineDifferenceIteratorT
assertDiff("a,b,d,e,f", "a,b,c,f, h", asList("c", "h"));
assertDiff("3,7", "2,3,5,9", asList("2", "5", "9"));
}
-
+
+ @Test
+ public void testMarkedDiffWithExtraEntriesInMarked() throws IOException {
+ assertReverseDiff("a,b,c,e,h", "a,b,c", asList("e", "h"));
+ assertReverseDiff("a,b,d,e", "a,b,c", asList("d", "e"));
+ assertReverseDiff("a,b,d", "a,b,d", Collections.<String>emptyList());
+ assertReverseDiff("a,0xb,d,e,f", "a,d", asList("e", "f"));
+ assertReverseDiff("a,0xb,d,e,f", "a,d,e,f,g",
Collections.<String>emptyList());
+ }
+
+ private static void assertReverseDiff(String marked, String all,
List<String> diff) throws IOException {
+ Iterator<String> itr = createItr(all, marked);
+ assertThat("marked: " + marked + " all: " + all,
ImmutableList.copyOf(itr), is(diff));
+ }
+
private static void assertDiff(String marked, String all, List<String>
diff) throws IOException {
Iterator<String> itr = createItr(marked, all);
assertThat("marked: " + marked + " all: " + all,
ImmutableList.copyOf(itr), is(diff));
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
Wed Jan 27 08:21:41 2016
@@ -25,6 +25,7 @@ import java.util.List;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.plugins.memory.PropertyBuilder;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -48,7 +49,7 @@ public class BlobCollectorTest {
@Test
public void testCollect() throws Exception {
NodeBuilder b1 = store.getRoot().builder();
- List<Blob> blobs = Lists.newArrayList();
+ List<ReferencedBlob> blobs = Lists.newArrayList();
b1.child("x").child("y");
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
@@ -58,7 +59,7 @@ public class BlobCollectorTest {
b1 = store.getRoot().builder();
Blob b = store.createBlob(randomStream(i, 4096));
b1.child("x").child("y").setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
@@ -68,7 +69,7 @@ public class BlobCollectorTest {
for(int i = 0; i < 2; i++){
Blob b = store.createBlob(randomStream(i, 4096));
p1.addValue(b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
}
b1 = store.getRoot().builder();
b1.child("x").child("y").setProperty(p1.getPropertyState());
@@ -80,16 +81,16 @@ public class BlobCollectorTest {
//Change the see to create diff binary
Blob b = store.createBlob(randomStream(i+1, 4096));
b1.child("x").child("y").setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
NodeDocument doc =
store.getDocumentStore().find(Collection.NODES,
Utils.getIdFromPath("/x/y"));
- List<Blob> collectedBlobs = Lists.newArrayList();
+ List<ReferencedBlob> collectedBlobs = Lists.newArrayList();
blobCollector.collect(doc, collectedBlobs);
assertEquals(blobs.size(), collectedBlobs.size());
- assertEquals(new HashSet<Blob>(blobs), new
HashSet<Blob>(collectedBlobs));
+ assertEquals(new HashSet<ReferencedBlob>(blobs), new
HashSet<ReferencedBlob>(collectedBlobs));
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
Wed Jan 27 08:21:41 2016
@@ -26,6 +26,7 @@ import java.util.List;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -76,19 +77,19 @@ public class BlobReferenceIteratorTest {
@Test
public void testBlobIterator() throws Exception{
- List<Blob> blobs = Lists.newArrayList();
+ List<ReferencedBlob> blobs = Lists.newArrayList();
//1. Set some single value Binary property
for(int i = 0; i < 10; i++){
NodeBuilder b1 = store.getRoot().builder();
Blob b = store.createBlob(randomStream(i, 4096));
b1.child("x").child("y"+1).setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y" + 1));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- List<Blob> collectedBlobs =
ImmutableList.copyOf(store.getReferencedBlobsIterator());
+ List<ReferencedBlob> collectedBlobs =
ImmutableList.copyOf(store.getReferencedBlobsIterator());
assertEquals(blobs.size(), collectedBlobs.size());
- assertEquals(new HashSet<Blob>(blobs), new
HashSet<Blob>(collectedBlobs));
+ assertEquals(new HashSet<ReferencedBlob>(blobs), new
HashSet<ReferencedBlob>(collectedBlobs));
}
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
Wed Jan 27 08:21:41 2016
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.Random;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -42,13 +43,13 @@ public class BlobReferenceTest {
HashSet<String> set = new HashSet<String>();
for (int i = 0; i < 100; i++) {
Blob b = a.createBlob(randomStream(i, 10));
- set.add(b.toString());
+ set.add(new ReferencedBlob(b, "/c" + i).toString());
a.child("c" + i).setProperty("x", b);
}
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- Iterator<Blob> it = s.getReferencedBlobsIterator();
+ Iterator<ReferencedBlob> it = s.getReferencedBlobsIterator();
while (it.hasNext()) {
- Blob b = it.next();
+ ReferencedBlob b = it.next();
set.remove(b.toString());
}
assertTrue(set.isEmpty());
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
Wed Jan 27 08:21:41 2016
@@ -97,6 +97,10 @@ public class MongoBlobGCTest extends Abs
}
}
a.child("c" + i).setProperty("x", b);
+ // Add a duplicated entry
+ if (i == 0) {
+ a.child("cdup").setProperty("x", b);
+ }
}
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
@@ -175,6 +179,7 @@ public class MongoBlobGCTest extends Abs
Set<String> existingAfterGC = gc(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
}
+
@Test
public void gcVersionDeleteWithInlined() throws Exception {
DataStoreState state = setUp(false);
@@ -184,6 +189,59 @@ public class MongoBlobGCTest extends Abs
}
@Test
+ public void consistencyCheckInlined() throws Exception {
+ DataStoreState state = setUp(true);
+ addInlined();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckInit() throws Exception {
+ DataStoreState state = setUp(true);
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithGc() throws Exception {
+ DataStoreState state = setUp(true);
+ Set<String> existingAfterGC = gc(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithRenegadeDelete() throws Exception {
+ DataStoreState state = setUp(true);
+
+ // Simulate faulty state by deleting some blobs directly
+ Random rand = new Random(87);
+ List<String> existing = Lists.newArrayList(state.blobsPresent);
+
+ GarbageCollectableBlobStore store = (GarbageCollectableBlobStore)
+
mk.getNodeStore().getBlobStore();
+ long count =
store.countDeleteChunks(ImmutableList.of(existing.get(rand.nextInt(existing.size()))),
0);
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(count, candidates);
+ }
+
+ @Test
public void gcLongRunningBlobCollection() throws Exception {
DataStoreState state = setUp(true);
log.info("{} Blobs added {}", state.blobsAdded.size(),
state.blobsAdded);
@@ -204,12 +262,21 @@ public class MongoBlobGCTest extends Abs
gc.collectGarbage(false);
Set<String> existingAfterGC = iterate();
log.info("{} Blobs existing after gc {}", existingAfterGC.size(),
existingAfterGC);
-
+
assertTrue(Sets.difference(state.blobsPresent,
existingAfterGC).isEmpty());
assertEquals(gc.additionalBlobs,
Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gc = init(blobGcMaxAgeInSecs, executor);
+ gc.collectGarbage(false);
+
+ assertEquals(0, executor.getTaskCount());
+ return iterate();
+ }
+
+ private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
DocumentNodeStore store = mk.getNodeStore();
String repoId = null;
@@ -219,14 +286,10 @@ public class MongoBlobGCTest extends Abs
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
- ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(), executor,
"./target", 5, blobGcMaxAgeInSecs, repoId);
- gc.collectGarbage(false);
-
- assertEquals(0, executor.getTaskCount());
- return iterate();
+ return gc;
}
protected Set<String> iterate() throws Exception {
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
Wed Jan 27 08:21:41 2016
@@ -108,7 +108,7 @@ public class ExternalBlobTest {
final List<String> refrences = Lists.newArrayList();
store.getTracker().collectBlobReferences(new ReferenceCollector() {
@Override
- public void addReference(String reference) {
+ public void addReference(String reference, String nodeId) {
assertNotNull(reference);
refrences.add(reference);
}
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java?rev=1726952&r1=1726951&r2=1726952&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
(original)
+++
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
Wed Jan 27 08:21:41 2016
@@ -22,7 +22,6 @@ import static org.apache.commons.io.File
import static
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -203,29 +202,47 @@ public class SegmentDataStoreBlobGCTest
Set<String> existingAfterGC = gcInternal(86400);
assertTrue(Sets.symmetricDifference(state.blobsAdded,
existingAfterGC).isEmpty());
}
-
- private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
- String repoId = null;
- if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
- repoId = ClusterRepositoryInfo.createId(nodeStore);
- ((SharedDataStore) store.getBlobStore()).addMetadataRecord(
- new ByteArrayInputStream(new byte[0]),
- REPOSITORY.getNameFromId(repoId));
- }
-
+
+ @Test
+ public void consistencyCheckInit() throws Exception {
+ DataStoreState state = setUp();
ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
- (GarbageCollectableBlobStore) store.getBlobStore(),
executor, "./target", 2048, maxBlobGcInSecs,
-
repoId);
- gc.collectGarbage(false);
-
- assertEquals(0, executor.getTaskCount());
- Set<String> existingAfterGC = iterate();
- log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
- return existingAfterGC;
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
}
-
+
+ @Test
+ public void consistencyCheckWithGc() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithRenegadeDelete() throws Exception {
+ DataStoreState state = setUp();
+
+ // Simulate faulty state by deleting some blobs directly
+ Random rand = new Random(87);
+ List<String> existing = Lists.newArrayList(state.blobsPresent);
+
+ long count =
blobStore.countDeleteChunks(ImmutableList.of(existing.get(rand.nextInt(existing.size()))),
0);
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(count, candidates);
+ }
+
@Test
public void gcLongRunningBlobCollection() throws Exception {
DataStoreState state = setUp();
@@ -250,7 +267,33 @@ public class SegmentDataStoreBlobGCTest
assertTrue(Sets.difference(state.blobsPresent,
existingAfterGC).isEmpty());
assertEquals(gc.additionalBlobs,
Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
-
+
+ private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gc = init(maxBlobGcInSecs, executor);
+ gc.collectGarbage(false);
+
+ assertEquals(0, executor.getTaskCount());
+ Set<String> existingAfterGC = iterate();
+ log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
+ return existingAfterGC;
+ }
+
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor) throws Exception {
+ String repoId = null;
+ if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
+ repoId = ClusterRepositoryInfo.createId(nodeStore);
+ ((SharedDataStore) store.getBlobStore()).addMetadataRecord(
+ new ByteArrayInputStream(new byte[0]),
+ REPOSITORY.getNameFromId(repoId));
+ }
+ MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ new SegmentBlobReferenceRetriever(store.getTracker()),
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor,
"./target", 2048, blobGcMaxAgeInSecs,
+ repoId);
+ return gc;
+ }
+
protected Set<String> iterate() throws Exception {
Iterator<String> cur = blobStore.getAllChunkIds(0);
