Author: amitj
Date: Fri Dec 14 06:01:43 2018
New Revision: 1848912
URL: http://svn.apache.org/viewvc?rev=1848912&view=rev
Log:
OAK-7951: Datastore GC stats not updated with failure when "Not all
repositories have marked references available"
Merge r1848822,r1848823 from trunk
Added:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
- copied, changed from r1848822,
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
Modified:
jackrabbit/oak/branches/1.8/ (props changed)
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Propchange: jackrabbit/oak/branches/1.8/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Dec 14 06:01:43 2018
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217
+/jackrabbit/oak/trunk:1820660-1820661,1820729,1820734,1820859,1820861,1820878,1820888,1820947,1821027,1821130,1821140-1821141,1821178,1821237,1821240,1821249,1821258,1821325,1821358,1821361-1821362,1821370,1821375,1821393,1821477,1821487,1821516,1821617,1821663,1821665,1821668,1821681,1821847,1821975-1821983,1822121,1822201,1822207,1822527,1822642,1822723,1822808,1822850,1822934,1823135,1823163,1823169,1823172,1823655,1823669,1824196,1824198,1824253,1824255,1824896,1824962,1825065,1825362,1825381,1825442,1825448,1825466,1825470-1825471,1825475,1825523,1825525,1825561,1825619-1825621,1825651,1825654,1825992,1826079,1826090,1826096,1826216,1826237,1826338,1826516,1826532,1826551,1826560,1826638,1826640,1826730,1826833,1826932,1826957,1827423,1827472,1827486,1827816,1827977,1828349,1828439,1828502,1828529,1828948,1829527,1829534,1829546,1829569,1829587,1829665,1829854,1829864,1829978,1829985,1829987,1829998,1830019,1830048,1830160,1830171,1830197,1830209,1830239,1830347,1830748,1830911
,1830923,1831157-1831158,1831163,1831190,1831374,1831560,1831689,1832258,1832376,1832379,1832535,1833308,1833347,1833833,1834112,1834117,1834287,1834291,1834302,1834326,1834328,1834336,1834428,1834468,1834483,1834610,1834648-1834649,1834681,1834823,1834857-1834858,1835060,1835518,1835521,1835635,1835642,1835780,1835819,1836082,1836121,1836167-1836168,1836170-1836187,1836189-1836196,1836206,1836487,1836493,1837057,1837274,1837296,1837326,1837475,1837503,1837547,1837569,1837600,1837657,1837718,1837998,1838076,1838637,1839549,1839570,1839637,1839746,1840019,1840024,1840031,1840226,1840455,1840462,1840574,1841314,1841352,1842089,1842677,1843175,1843222,1843231,1843398,1843618,1843652,1843911,1844325,1844549,1844625,1844627,1844642,1844728,1844775,1844932,1845135,1845336,1845405,1845415,1845730-1845731,1845863,1845865,1846057,1846617,1848073,1848181-1848182,1848191,1848217,1848822-1848823
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1848912&r1=1848911&r2=1848912&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Fri Dec 14 06:01:43 2018
@@ -77,10 +77,10 @@ import static org.apache.jackrabbit.oak.
/**
* Mark and sweep garbage collector.
- *
+ *
* Uses the file system to store internal state while in process to account
for huge data.
* This class is not thread safe.
- *
+ *
*/
public class MarkSweepGarbageCollector implements BlobGarbageCollector {
@@ -89,7 +89,7 @@ public class MarkSweepGarbageCollector i
public static final String TEMP_DIR =
StandardSystemProperty.JAVA_IO_TMPDIR.value();
public static final int DEFAULT_BATCH_COUNT = 1024;
-
+
public static final String DELIM = ",";
private static final Function<String, String> transformer = new
Function<String, String>() {
@@ -200,7 +200,7 @@ public class MarkSweepGarbageCollector i
/**
* Returns the stats related to GC for all repos
- *
+ *
* @return a list of GarbageCollectionRepoStats objects
* @throws Exception
*/
@@ -212,12 +212,12 @@ public class MarkSweepGarbageCollector i
List<DataRecord> refFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
Map<String, DataRecord> references = Maps.uniqueIndex(refFiles,
new Function<DataRecord, String>() {
- @Override
+ @Override
public String apply(DataRecord input) {
return
SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString());
}
});
-
+
// Get all the markers available
List<DataRecord> markerFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType());
@@ -227,11 +227,11 @@ public class MarkSweepGarbageCollector i
return
SharedStoreRecordType.MARKED_START_MARKER.getIdFromName(input.getIdentifier().toString());
}
});
-
+
// Get all the repositories registered
List<DataRecord> repoFiles =
((SharedDataStore)
blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
-
+
for (DataRecord repoRec : repoFiles) {
String id =
SharedStoreRecordType.REFERENCES.getIdFromName(repoRec.getIdentifier().toString());
GarbageCollectionRepoStats stat = new
GarbageCollectionRepoStats();
@@ -244,11 +244,11 @@ public class MarkSweepGarbageCollector i
DataRecord refRec = references.get(id);
stat.setEndTime(refRec.getLastModified());
stat.setLength(refRec.getLength());
-
+
if (markers.containsKey(id)) {
stat.setStartTime(markers.get(id).getLastModified());
}
-
+
LineNumberReader reader = null;
try {
reader = new LineNumberReader(new
InputStreamReader(refRec.getStream()));
@@ -279,7 +279,7 @@ public class MarkSweepGarbageCollector i
try {
Stopwatch sw = Stopwatch.createStarted();
LOG.info("Starting Blob garbage collection with markOnly [{}]",
markOnly);
-
+
long markStart = System.currentTimeMillis();
mark(fs);
if (!markOnly) {
@@ -323,7 +323,7 @@ public class MarkSweepGarbageCollector i
/**
* Difference phase where the GC candidates are identified.
- *
+ *
* @param fs the garbage collector file state
* @throws IOException
* Signals that an I/O exception has occurred.
@@ -382,14 +382,10 @@ public class MarkSweepGarbageCollector i
long earliestRefAvailTime;
// Merge all the blob references available from all the reference
files in the data store meta store
// Only go ahead if merge succeeded
- try {
- earliestRefAvailTime =
-
GarbageCollectionType.get(blobStore).mergeAllMarkedReferences(blobStore, fs);
- LOG.debug("Earliest reference available for timestamp [{}]",
earliestRefAvailTime);
- earliestRefAvailTime = (earliestRefAvailTime < markStart ?
earliestRefAvailTime : markStart);
- } catch (Exception e) {
- return 0;
- }
+ earliestRefAvailTime =
+
GarbageCollectionType.get(blobStore).mergeAllMarkedReferences(blobStore, fs);
+ LOG.debug("Earliest reference available for timestamp [{}]",
earliestRefAvailTime);
+ earliestRefAvailTime = (earliestRefAvailTime < markStart ?
earliestRefAvailTime : markStart);
// Find all blob references after iterating over the whole repository
(new BlobIdRetriever(fs, forceBlobRetrieve)).call();
@@ -398,7 +394,7 @@ public class MarkSweepGarbageCollector i
difference(fs);
long count = 0;
long deleted = 0;
-
+
long maxModifiedTime = getMaxModifiedTime(earliestRefAvailTime);
LOG.debug("Starting sweep phase of the garbage collector");
LOG.debug("Sweeping blobs with modified time > than the configured max
deleted time ({}). ",
@@ -440,7 +436,7 @@ public class MarkSweepGarbageCollector i
BlobCollectionType.get(blobStore).handleRemoves(blobStore,
fs.getGarbage(), fs.getMarkedRefs());
if(count != deleted) {
- LOG.warn("Deleted only [{}] blobs entries from the [{}] candidates
identified. This may happen if blob "
+ LOG.warn("Deleted only [{}] blobs entries from the [{}] candidates
identified. This may happen if blob "
+ "modified time is > "
+ "than the max deleted time ({})", deleted, count,
timestampToString(maxModifiedTime));
@@ -568,10 +564,10 @@ public class MarkSweepGarbageCollector i
closeQuietly(writer);
}
}
-
+
/**
* Checks for the DataStore consistency and reports the number of missing
blobs still referenced.
- *
+ *
* @return the missing blobs
* @throws Exception
*/
@@ -580,18 +576,18 @@ public class MarkSweepGarbageCollector i
boolean threw = true;
GarbageCollectorFileState fs = new GarbageCollectorFileState(root);
long candidates = 0;
-
+
try {
LOG.info("Starting blob consistency check");
-
+
// Find all blobs available in the blob store
ListenableFutureTask<Integer> blobIdRetriever =
ListenableFutureTask.create(new BlobIdRetriever(fs,
true));
executor.execute(blobIdRetriever);
-
+
// Mark all used blob references
iterateNodeTree(fs, true);
-
+
try {
blobIdRetriever.get();
} catch (ExecutionException e) {
@@ -599,7 +595,7 @@ public class MarkSweepGarbageCollector i
threw = false;
throw e;
}
-
+
LOG.trace("Starting difference phase of the consistency check");
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getAvailableRefs(),
@@ -634,7 +630,7 @@ public class MarkSweepGarbageCollector i
this.fs = fs;
this.forceRetrieve = forceBlobRetrieve;
}
-
+
@Override
public Integer call() throws Exception {
if (!forceRetrieve) {
@@ -669,7 +665,7 @@ public class MarkSweepGarbageCollector i
SHARED {
/**
* Remove the maked references and the marked markers from the
blob store root. Default NOOP.
- *
+ *
* @param blobStore the blobStore instance
*/
@Override
@@ -680,7 +676,7 @@ public class MarkSweepGarbageCollector i
/**
* Merge all marked references available from all repositories and
return the earliest time of the references.
- *
+ *
* @param blobStore the blob store
* @param fs the fs
* @return the long the earliest time of the available references
@@ -711,7 +707,7 @@ public class MarkSweepGarbageCollector i
}
merge(files, fs.getMarkedRefs());
-
+
// Get the timestamp to indicate the earliest mark phase
start
List<DataRecord> markerFiles =
((SharedDataStore) blobStore).getAllMetadataRecords(
@@ -731,7 +727,7 @@ public class MarkSweepGarbageCollector i
/**
* Adds the marked references to the blob store root. Default NOOP
- *
+ *
* @param blobStore the blob store
* @param fs the fs
* @param repoId the repo id
@@ -744,7 +740,7 @@ public class MarkSweepGarbageCollector i
((SharedDataStore) blobStore)
.addMetadataRecord(fs.getMarkedRefs(),
SharedStoreRecordType.REFERENCES.getNameFromId(repoId));
}
-
+
@Override
public void addMarkedStartMarker(GarbageCollectableBlobStore
blobStore, String repoId) {
try {
@@ -779,7 +775,7 @@ public class MarkSweepGarbageCollector i
}
return DEFAULT;
}
-
+
public void addMarkedStartMarker(GarbageCollectableBlobStore
blobStore, String repoId) {}
}
Copied:
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
(from r1848822,
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java?p2=jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java&p1=jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java&r1=1848822&r2=1848912&rev=1848912&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/SharedDataStoreMarkSweepGarbageCollectorTest.java
Fri Dec 14 06:01:43 2018
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
package org.apache.jackrabbit.oak.plugins.blob;
import com.google.common.collect.ImmutableList;
@@ -5,13 +24,11 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean;
-import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.whiteboard.Tracker;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
-import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
-import org.junit.After;
+import org.hamcrest.CoreMatchers;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -23,12 +40,9 @@ import org.mockito.junit.MockitoJUnitRun
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Executor;
import static
org.apache.jackrabbit.oak.plugins.blob.SharedDataStore.Type.SHARED;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -36,6 +50,9 @@ import static org.mockito.Mockito.when;
public class SharedDataStoreMarkSweepGarbageCollectorTest {
@Mock
+ private Executor executor;
+
+ @Mock
private MockGarbageCollectableSharedDataStore blobStore;
@Mock
@@ -55,8 +72,6 @@ public class SharedDataStoreMarkSweepGar
@Mock
private CheckpointMBean checkpointMBean;
- private ScheduledExecutorService executor =
Executors.newSingleThreadScheduledExecutor();
-
@Before
public void setUp() throws IOException {
when(whiteboard.track(CheckpointMBean.class)).thenReturn(tracker);
@@ -72,24 +87,18 @@ public class SharedDataStoreMarkSweepGar
1,
0L,
"repo",
- whiteboard,
- new DefaultStatisticsProvider(executor)
+ whiteboard
);
}
- @After
- public void tear() {
- new ExecutorCloser(executor).close();
- }
-
@Test
public void
markAndSweepShouldFailIfNotAllRepositoriesHaveMarkedReferencesAvailable()
throws Exception {
+ exception.expect(IOException.class);
+ exception.expectMessage(CoreMatchers.containsString("Not all repositories
have marked references available"));
+
setupSharedDataRecords("REPO1", "REPO2");
collector.markAndSweep(false, true);
-
- assertThat(collector.getOperationStats().numDeleted(), is(0L));
- assertThat(collector.getOperationStats().getFailureCount(), is(1L));
}
@Test
@@ -98,9 +107,6 @@ public class SharedDataStoreMarkSweepGar
when(blobStore.getAllChunkIds(0L)).thenReturn(ImmutableList.<String>of().iterator());
collector.markAndSweep(false, true);
-
- assertThat(collector.getOperationStats().numDeleted(), is(0L));
- assertThat(collector.getOperationStats().getFailureCount(), is(0L));
}
private void setupSharedDataRecords(final String refRepoId, final String
repoRepoId) throws DataStoreException {
Modified:
jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1848912&r1=1848911&r2=1848912&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Fri Dec 14 06:01:43 2018
@@ -56,10 +56,12 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.stats.Clock;
+import org.hamcrest.CoreMatchers;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,6 +75,9 @@ public class SharedBlobStoreGCTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
protected Cluster cluster1;
protected Cluster cluster2;
private Clock clock;
@@ -190,6 +195,9 @@ public class SharedBlobStoreGCTest {
@Test
// GC should fail
public void testOnly1ClusterMark() throws Exception {
+ exception.expect(IOException.class);
+ exception.expectMessage(CoreMatchers.containsString("Not all
repositories have marked references available"));
+
log.debug("Running testOnly1ClusterMark()");
// Only run the mark phase on one cluster
@@ -197,12 +205,6 @@ public class SharedBlobStoreGCTest {
// Execute the gc with sweep
cluster1.gc.collectGarbage(false);
-
- Set<String> existing = cluster1.getExistingBlobIds();
- log.debug("Existing blobs {}", existing);
- Assert.assertTrue((cluster1.getInitBlobs().size() +
cluster2.getInitBlobs().size()) <= existing.size());
- Assert.assertTrue(existing.containsAll(cluster2.getInitBlobs()));
- Assert.assertTrue(existing.containsAll(cluster1.getInitBlobs()));
}
@Test
