Author: amitj
Date: Thu May 16 06:52:07 2019
New Revision: 1859351
URL: http://svn.apache.org/viewvc?rev=1859351&view=rev
Log:
OAK-8314: BlobIds created with direct uploads lack length suffix
Enable DocumentNodeStore-Mongo fixture for BinaryAccessDSGCIT
Added:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
- copied, changed from r1859350,
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java?rev=1859351&r1=1859350&r2=1859351&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/BinaryAccessDSGCIT.java
Thu May 16 06:52:07 2019
@@ -56,25 +56,33 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.AzureDataStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.S3DataStoreFixture;
+import
org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.DocumentMongoNodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.nodestore.SegmentMemoryNodeStoreFixture;
+import org.apache.jackrabbit.oak.jcr.binary.util.BinaryAccessDSGCFixture;
import org.apache.jackrabbit.oak.jcr.binary.util.Content;
+import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
-import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
import org.junit.runners.Parameterized;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class BinaryAccessDSGCIT extends AbstractBinaryAccessIT {
+ private static Logger LOG =
LoggerFactory.getLogger(BinaryAccessDSGCIT.class);
+
private static final String TEST_ROOT = "testroot";
private static final long BINARY_SIZE = 1024*1024;
@@ -83,17 +91,24 @@ public class BinaryAccessDSGCIT extends
private static final String DIRECT_UPLOAD_1 = "du1";
private static final String DIRECT_UPLOAD_2 = "du2";
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
@Parameterized.Parameters(name = "{0}")
public static Iterable<?> dataStoreFixtures() {
Collection<NodeStoreFixture> fixtures = new ArrayList<>();
fixtures.add(new SegmentMemoryNodeStoreFixture(new
S3DataStoreFixture()));
+ fixtures.add(new DocumentMongoNodeStoreFixture(new
S3DataStoreFixture()));
+
fixtures.add(new SegmentMemoryNodeStoreFixture(new
AzureDataStoreFixture()));
+ fixtures.add(new DocumentMongoNodeStoreFixture(new
AzureDataStoreFixture()));
+
return fixtures;
}
public BinaryAccessDSGCIT(NodeStoreFixture fixture) {
// reuse NodeStore (and DataStore) across all tests in this class
- super(fixture, true);
+ super(fixture, false);
}
private Session session;
@@ -190,8 +205,6 @@ public class BinaryAccessDSGCIT extends
private MarkSweepGarbageCollector getGarbageCollector()
throws DataStoreException, IOException {
DataStoreBlobStore blobStore = (DataStoreBlobStore)
getNodeStoreComponent(BlobStore.class);
- FileStore fileStore = getNodeStoreComponent(FileStore.class);
- File fileStoreRoot = getNodeStoreComponent(FileStore.class.getName() +
":root");
if (null == garbageCollector) {
String repoId =
ClusterRepositoryInfo.getOrCreateId(getNodeStore());
@@ -200,11 +213,12 @@ public class BinaryAccessDSGCIT extends
if (null == executor) {
executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
}
+ BlobReferenceRetriever referenceRetriever =
((BinaryAccessDSGCFixture) fixture).getBlobReferenceRetriever(getNodeStore());
garbageCollector = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(fileStore),
+ referenceRetriever,
blobStore,
executor,
- fileStoreRoot.getAbsolutePath(),
+ folder.newFolder().getAbsolutePath(),
2048,
0,
repoId
@@ -225,6 +239,8 @@ public class BinaryAccessDSGCIT extends
@Test
public void testGC() throws Exception {
+ LOG.info("Starting testGC [{}]", fixture);
+
Map<String, Content> binaryContent = Maps.newHashMap();
Map<String, Binary> binaries = Maps.newHashMap();
@@ -256,7 +272,6 @@ public class BinaryAccessDSGCIT extends
}
session.save();
-
// Verify that they are deleted from repo
for (String path : deletedBinaryPaths) {
assertFalse(session.nodeExists(toAbsolutePath(path)));
@@ -267,7 +282,7 @@ public class BinaryAccessDSGCIT extends
// Run DSGC
- compactFileStore();
+ ((BinaryAccessDSGCFixture) fixture).compactStore(getNodeStore());
MarkSweepGarbageCollector garbageCollector = getGarbageCollector();
garbageCollector.collectGarbage(false);
Copied:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
(from r1859350,
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java?p2=jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java&p1=jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java&r1=1859350&r2=1859351&rev=1859351&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/DocumentMongoNodeStoreFixture.java
Thu May 16 06:52:07 2019
@@ -20,60 +20,74 @@ package org.apache.jackrabbit.oak.jcr.bi
import java.io.File;
import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
import javax.jcr.RepositoryException;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Table;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.DataStoreFixture;
+import org.apache.jackrabbit.oak.jcr.binary.util.BinaryAccessDSGCFixture;
import org.apache.jackrabbit.oak.jcr.util.ComponentHolder;
+import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
-import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import org.apache.jackrabbit.oak.segment.file.FileStore;
-import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
-import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
+import
org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
+import org.apache.jackrabbit.oak.plugins.document.Revision;
+import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
+import
org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentNodeStoreBuilder;
+import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.HashBasedTable;
-import com.google.common.collect.Table;
-
/**
* Creates a repository with
- * - SegmentNodeStore, storing data in-memory
+ * - DocumentNodeStore, storing data in mongo (available locally or deloying
in docker if available)
* - an optional DataStore provided by DataStoreFixture
*/
-public class SegmentMemoryNodeStoreFixture extends NodeStoreFixture implements
ComponentHolder {
+public class DocumentMongoNodeStoreFixture extends NodeStoreFixture implements
ComponentHolder,
+ BinaryAccessDSGCFixture {
private final Logger log = LoggerFactory.getLogger(getClass());
private final DataStoreFixture dataStoreFixture;
-
private final Table<NodeStore, String, Object> components =
HashBasedTable.create();
+ private MongoConnection connection;
+ private final Clock clock;
+ public final MongoConnectionFactory connFactory = new
MongoConnectionFactory();
- public SegmentMemoryNodeStoreFixture(@Nullable DataStoreFixture
dataStoreFixture) {
+ public DocumentMongoNodeStoreFixture(@Nullable DataStoreFixture
dataStoreFixture) {
this.dataStoreFixture = dataStoreFixture;
+ this.clock = new Clock.Virtual();
}
@Override
public boolean isAvailable() {
+ this.connection = connFactory.getConnection();
+
// if a DataStore is configured, it must be available for our
NodeStore to be available
- return dataStoreFixture == null || dataStoreFixture.isAvailable();
+ return (dataStoreFixture == null || dataStoreFixture.isAvailable()) &&
(connection != null);
}
@Override
public NodeStore createNodeStore() {
+
try {
log.info("Creating NodeStore using " + toString());
+ clock.waitUntil(Revision.getCurrentTimestamp());
- File fileStoreRoot = FixtureUtils.createTempFolder();
- FileStoreBuilder fileStoreBuilder =
FileStoreBuilder.fileStoreBuilder(fileStoreRoot)
- .withNodeDeduplicationCacheSize(16384)
- .withMaxFileSize(256)
- .withMemoryMapping(false);
+ MongoDocumentNodeStoreBuilder documentNodeStoreBuilder =
+
MongoDocumentNodeStoreBuilder.newMongoDocumentNodeStoreBuilder()
+ .setMongoDB(connection.getMongoClient(),
connection.getDBName());
+ documentNodeStoreBuilder.clock(clock);
File dataStoreFolder = null;
BlobStore blobStore = null;
@@ -86,11 +100,10 @@ public class SegmentMemoryNodeStoreFixtu
dataStore.init(dataStoreFolder.getAbsolutePath());
blobStore = new DataStoreBlobStore(dataStore);
- fileStoreBuilder.withBlobStore(blobStore);
+ documentNodeStoreBuilder.setBlobStore(blobStore);
}
- FileStore fileStore = fileStoreBuilder.build();
- NodeStore nodeStore =
SegmentNodeStoreBuilders.builder(fileStore).build();
+ NodeStore nodeStore = documentNodeStoreBuilder.build();
// track all main components
if (dataStore != null) {
@@ -100,12 +113,9 @@ public class SegmentMemoryNodeStoreFixtu
if (blobStore != null) {
components.put(nodeStore, BlobStore.class.getName(),
blobStore);
}
- components.put(nodeStore, FileStore.class.getName(), fileStore);
- components.put(nodeStore, FileStore.class.getName() + ":root",
fileStoreRoot);
return nodeStore;
-
- } catch (IOException | InvalidFileStoreVersionException |
RepositoryException e) {
+ } catch (IOException | RepositoryException | InterruptedException e) {
throw new AssertionError("Cannot create test repo fixture " +
toString(), e);
}
}
@@ -113,8 +123,9 @@ public class SegmentMemoryNodeStoreFixtu
@Override
public void dispose(NodeStore nodeStore) {
try {
- File fileStoreRoot = (File) components.get(nodeStore,
FileStore.class.getName() + ":root");
- FileUtils.deleteQuietly(fileStoreRoot);
+ if (nodeStore instanceof DocumentNodeStore) {
+ ((DocumentNodeStore)nodeStore).dispose();
+ }
DataStore dataStore = (DataStore) components.get(nodeStore,
DataStore.class.getName());
if (dataStore != null && dataStoreFixture != null) {
@@ -123,12 +134,25 @@ public class SegmentMemoryNodeStoreFixtu
File dataStoreFolder = (File) components.get(nodeStore,
DataStore.class.getName() + ":folder");
FileUtils.deleteQuietly(dataStoreFolder);
}
+ connection.close();
} finally {
components.row(nodeStore).clear();
}
}
@Override
+ public void compactStore(NodeStore nodeStore) throws IOException,
InterruptedException {
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(10));
+ VersionGarbageCollector vGC = ((DocumentNodeStore)
nodeStore).getVersionGarbageCollector();
+ VersionGarbageCollector.VersionGCStats stats = vGC.gc(0,
TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public BlobReferenceRetriever getBlobReferenceRetriever(NodeStore
nodeStore) {
+ return new DocumentBlobReferenceRetriever((DocumentNodeStore)
nodeStore);
+ }
+
+ @Override
public String toString() {
// for nice Junit parameterized test labels
return FixtureUtils.getFixtureLabel(this, dataStoreFixture);
Modified:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java?rev=1859351&r1=1859350&r2=1859351&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/fixtures/nodestore/SegmentMemoryNodeStoreFixture.java
Thu May 16 06:52:07 2019
@@ -26,9 +26,13 @@ import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.fixture.NodeStoreFixture;
import
org.apache.jackrabbit.oak.jcr.binary.fixtures.datastore.DataStoreFixture;
+import org.apache.jackrabbit.oak.jcr.binary.util.BinaryAccessDSGCFixture;
import org.apache.jackrabbit.oak.jcr.util.ComponentHolder;
+import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
@@ -46,7 +50,8 @@ import com.google.common.collect.Table;
* - SegmentNodeStore, storing data in-memory
* - an optional DataStore provided by DataStoreFixture
*/
-public class SegmentMemoryNodeStoreFixture extends NodeStoreFixture implements
ComponentHolder {
+public class SegmentMemoryNodeStoreFixture extends NodeStoreFixture implements
ComponentHolder,
+ BinaryAccessDSGCFixture {
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -129,6 +134,19 @@ public class SegmentMemoryNodeStoreFixtu
}
@Override
+ public void compactStore(NodeStore nodeStore) {
+ FileStore fileStore = get(nodeStore, FileStore.class.getName());
+ for (int i = 0; i<
SegmentGCOptions.defaultGCOptions().getRetainedGenerations(); i++) {
+ fileStore.compactFull();
+ }
+ }
+
+ @Override
+ public BlobReferenceRetriever getBlobReferenceRetriever(NodeStore
nodeStore) {
+ return new SegmentBlobReferenceRetriever(get(nodeStore,
FileStore.class.getName()));
+ }
+
+ @Override
public String toString() {
// for nice Junit parameterized test labels
return FixtureUtils.getFixtureLabel(this, dataStoreFixture);
Added:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java?rev=1859351&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
(added)
+++
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
Thu May 16 06:52:07 2019
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.binary.util;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+/** Marker Interface for DSGC Fixture */
+public interface BinaryAccessDSGCFixture {
+
+ void compactStore(NodeStore nodeStore) throws IOException,
InterruptedException;
+
+ BlobReferenceRetriever getBlobReferenceRetriever(NodeStore nodeStore);
+}
Propchange:
jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/binary/util/BinaryAccessDSGCFixture.java
------------------------------------------------------------------------------
svn:eol-style = native
