Author: amitj
Date: Thu Aug 10 03:52:24 2017
New Revision: 1804626

URL: http://svn.apache.org/viewvc?rev=1804626&view=rev
Log:
OAK-6504: Active deletion of blobs needs to indicate information about purged 
blobs to mark-sweep collector

- Synchronizing the blob tracker with the deleted blob ids
- Added an integration test with FDS

Added:
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
   (with props)
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
   (with props)
Modified:
    
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
    
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java?rev=1804626&r1=1804625&r2=1804626&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectorFactory.java
 Thu Aug 10 03:52:24 2017
@@ -16,12 +16,18 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
 
+import com.google.common.base.Charsets;
 import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
+import com.google.common.io.Closeables;
+import com.google.common.io.Files;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.io.filefilter.RegexFileFilter;
+import org.apache.jackrabbit.oak.commons.FileIOUtils;
 import org.apache.jackrabbit.oak.commons.benchmark.PerfLogger;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobTracker;
 import org.apache.jackrabbit.oak.plugins.index.IndexCommitCallback;
 import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
 import org.apache.jackrabbit.oak.stats.Clock;
@@ -31,6 +37,7 @@ import org.slf4j.LoggerFactory;
 import javax.annotation.Nonnull;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -161,6 +168,21 @@ public class ActiveDeletedBlobCollectorF
             long numBlobsDeleted = 0;
             long numChunksDeleted = 0;
 
+            File idTempDeleteFile = null;
+            BufferedWriter idTempDeleteWriter = null;
+            // If blob store support blob tracking
+            boolean blobIdsTracked = blobStore instanceof BlobTrackingStore;
+
+            if (blobIdsTracked) {
+                try {
+                    idTempDeleteFile = File.createTempFile("idTempDelete", 
null, rootDirectory);
+                    idTempDeleteWriter = Files.newWriter(idTempDeleteFile, 
Charsets.UTF_8);
+                } catch (Exception e) {
+                    LOG.warn("Unable to open a writer to a temp file, will 
ignore tracker sync");
+                    blobIdsTracked = false;
+                }
+            }
+
             long lastCheckedBlobTimestamp = readLastCheckedBlobTimestamp();
             String currInUseFileName = deletedBlobsFileWriter.inUseFileName;
             deletedBlobsFileWriter.releaseInUseFile();
@@ -204,6 +226,13 @@ public class ActiveDeletedBlobCollectorF
                                         } else {
                                             numBlobsDeleted++;
                                             numChunksDeleted += deleted;
+
+                                            if (blobIdsTracked) {
+                                                // Save deleted chunkIds to a 
temporary file
+                                                for (String id : chunkIds) {
+                                                    
FileIOUtils.writeAsLine(idTempDeleteWriter, id, true);
+                                                }
+                                            }
                                         }
                                     }
                                 } catch (NumberFormatException nfe) {
@@ -230,6 +259,21 @@ public class ActiveDeletedBlobCollectorF
                     LOG.debug("Skipping {} as its timestamp is newer than {}", 
deletedBlobListFile.getName(), before);
                 }
             }
+
+            // Synchronize deleted blob ids with the blob id tracker
+            try {
+                Closeables.close(idTempDeleteWriter, true);
+
+                if (blobIdsTracked && numBlobsDeleted > 0) {
+                    BlobTracker tracker = ((BlobTrackingStore) 
blobStore).getTracker();
+                    if (tracker != null) {
+                        tracker.remove(idTempDeleteFile);
+                    }
+                }
+            } catch(Exception e) {
+                LOG.warn("Error refreshing tracked blob ids", e);
+            }
+
             LOG.info("Deleted {} blobs contained in {} chunks", 
numBlobsDeleted, numChunksDeleted);
             writeOutLastCheckedBlobTimestamp(before);
         }

Added: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java?rev=1804626&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
 Thu Aug 10 03:52:24 2017
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.Iterators;
+import org.apache.jackrabbit.JcrConstants;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataRecord;
+import org.apache.jackrabbit.core.data.DataStoreException;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobTracker;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
+import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
+import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
+import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+
+public abstract class AbstractActiveDeletedBlobTest extends AbstractQueryTest {
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder(new 
File("target"));
+    @Rule
+    public TemporaryFolder blobCollectionRoot = new TemporaryFolder(new 
File("target"));
+    @Rule
+    public TemporaryFolder fileDataStoreRoot = new TemporaryFolder(new 
File("target"));
+
+    protected ExecutorService executorService = 
Executors.newFixedThreadPool(2);
+
+    protected CountingBlobStore blobStore = null;
+
+    protected Clock clock = new Clock.Virtual();
+
+    protected ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollectorImpl 
adbc = null;
+
+    protected AsyncIndexUpdate asyncIndexUpdate;
+
+    protected LuceneIndexEditorProvider editorProvider;
+
+    protected NodeStore nodeStore;
+
+    protected LuceneIndexProvider provider;
+
+    @After
+    public void after() {
+        new ExecutorCloser(executorService).close();
+        executorService.shutdown();
+        IndexDefinition.setDisableStoredIndexDefinition(false);
+    }
+
+    public static Tree createIndex(Tree index, String name, Set<String> 
propNames) throws CommitFailedException {
+        Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name);
+        def.setProperty(JcrConstants.JCR_PRIMARYTYPE,
+                INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
+        def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
+        def.setProperty(REINDEX_PROPERTY_NAME, true);
+        def.setProperty(ASYNC_PROPERTY_NAME, "async");
+        def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false);
+        
def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES,
 propNames, Type.STRINGS));
+        def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true);
+        return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name);
+    }
+
+    @Override
+    protected void createTestIndexNode() throws Exception {
+        setTraversalEnabled(false);
+    }
+
+    protected IndexCopier createIndexCopier() {
+        try {
+            return new IndexCopier(executorService, temporaryFolder.getRoot());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected Tree createIndex(String name, Set<String> propNames) throws 
CommitFailedException {
+        Tree index = root.getTree("/");
+        return createIndex(index, name, propNames);
+    }
+
+    class CountingBlobStore implements GarbageCollectableBlobStore, 
BlobTrackingStore {
+
+        private final GarbageCollectableBlobStore delegate;
+        protected long numChunks = 0;
+
+        CountingBlobStore(GarbageCollectableBlobStore delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override public String writeBlob(InputStream in) throws IOException {
+            String blobId = delegate.writeBlob(in);
+            numChunks += Iterators.size(delegate.resolveChunks(blobId));
+            return blobId;
+        }
+
+        @Override public void setBlockSize(int x) {
+            delegate.setBlockSize(x);
+        }
+
+        @Override public String writeBlob(InputStream in, BlobOptions options) 
throws IOException {
+            String blobId = delegate.writeBlob(in, options);
+            numChunks += Iterators.size(delegate.resolveChunks(blobId));
+            return blobId;
+        }
+
+        @Override public String writeBlob(String tempFileName) throws 
IOException {
+            String blobId = delegate.writeBlob(tempFileName);
+            numChunks += Iterators.size(delegate.resolveChunks(blobId));
+            return blobId;
+        }
+
+        @Override public int sweep() throws IOException {
+            return delegate.sweep();
+        }
+
+        @Override public void startMark() throws IOException {
+            delegate.startMark();
+        }
+
+        @Override public int readBlob(String blobId, long pos, byte[] buff, 
int off, int length) throws IOException {
+            return delegate.readBlob(blobId, pos, buff, off, length);
+        }
+
+        @Override public void clearInUse() {
+            delegate.clearInUse();
+        }
+
+        @Override public void clearCache() {
+            delegate.clearCache();
+        }
+
+        @Override public long getBlobLength(String blobId) throws IOException {
+            return delegate.getBlobLength(blobId);
+        }
+
+        @Override public long getBlockSizeMin() {
+            return delegate.getBlockSizeMin();
+        }
+
+        @Override public Iterator<String> getAllChunkIds(long 
maxLastModifiedTime) throws Exception {
+            return delegate.getAllChunkIds(maxLastModifiedTime);
+        }
+
+        @Override public InputStream getInputStream(String blobId) throws 
IOException {
+            return delegate.getInputStream(blobId);
+        }
+
+        @Override @Deprecated public boolean deleteChunks(List<String> 
chunkIds, long maxLastModifiedTime)
+            throws Exception {
+            numChunks -= chunkIds.size();
+            return delegate.deleteChunks(chunkIds, maxLastModifiedTime);
+        }
+
+        @Override @CheckForNull public String getBlobId(@Nonnull String 
reference) {
+            return delegate.getBlobId(reference);
+        }
+
+        @Override @CheckForNull public String getReference(@Nonnull String 
blobId) {
+            return delegate.getReference(blobId);
+        }
+
+        @Override public long countDeleteChunks(List<String> chunkIds, long 
maxLastModifiedTime) throws Exception {
+            long numDeleted = delegate.countDeleteChunks(chunkIds, 
maxLastModifiedTime);
+            numChunks -= numDeleted;
+            return numDeleted;
+        }
+
+        @Override public Iterator<String> resolveChunks(String blobId) throws 
IOException {
+            return delegate.resolveChunks(blobId);
+        }
+
+        @Override public void addTracker(BlobTracker tracker) {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) delegate).addTracker(tracker);
+            }
+        }
+
+        @Override public BlobTracker getTracker() {
+            if (delegate instanceof BlobTrackingStore) {
+                return ((BlobTrackingStore) delegate).getTracker();
+            }
+            return null;
+        }
+
+        @Override public void addMetadataRecord(InputStream stream, String 
name) throws DataStoreException {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) delegate).addMetadataRecord(stream, name);
+            }
+        }
+
+        @Override public void addMetadataRecord(File f, String name) throws 
DataStoreException {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) delegate).addMetadataRecord(f, name);
+            }
+        }
+
+        @Override public DataRecord getMetadataRecord(String name) {
+            if (delegate instanceof BlobTrackingStore) {
+                return ((BlobTrackingStore) delegate).getMetadataRecord(name);
+            }
+            return null;
+        }
+
+        @Override public List<DataRecord> getAllMetadataRecords(String prefix) 
{
+            if (delegate instanceof BlobTrackingStore) {
+                return ((BlobTrackingStore) 
delegate).getAllMetadataRecords(prefix);
+            }
+            return null;
+        }
+
+        @Override public boolean deleteMetadataRecord(String name) {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) delegate).deleteMetadataRecord(name);
+            }
+            return false;
+        }
+
+        @Override public void deleteAllMetadataRecords(String prefix) {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) 
delegate).deleteAllMetadataRecords(prefix);
+            }
+        }
+
+        @Override public Iterator<DataRecord> getAllRecords() throws 
DataStoreException {
+            if (delegate instanceof BlobTrackingStore) {
+                return ((BlobTrackingStore) delegate).getAllRecords();
+            }
+            return Iterators.emptyIterator();
+        }
+
+        @Override public DataRecord getRecordForId(DataIdentifier id) throws 
DataStoreException {
+            if (delegate instanceof BlobTrackingStore) {
+                return ((BlobTrackingStore) delegate).getRecordForId(id);
+            }
+            return null;
+        }
+
+        @Override public Type getType() {
+            if (delegate instanceof BlobTrackingStore) {
+                ((BlobTrackingStore) delegate).getType();
+            }
+            return Type.DEFAULT;
+        }
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/AbstractActiveDeletedBlobTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java?rev=1804626&r1=1804625&r2=1804626&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java
 Thu Aug 10 03:52:24 2017
@@ -16,18 +16,12 @@
  */
 package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
 
-import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import org.apache.commons.io.FileUtils;
-import org.apache.jackrabbit.JcrConstants;
 import org.apache.jackrabbit.core.data.FileDataStore;
 import org.apache.jackrabbit.oak.InitialContent;
 import org.apache.jackrabbit.oak.Oak;
-import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.api.ContentRepository;
-import org.apache.jackrabbit.oak.api.Tree;
-import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
 import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
 import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
@@ -37,76 +31,35 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
 import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
 import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
-import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollectorImpl;
-import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
-import org.apache.jackrabbit.oak.query.AbstractQueryTest;
-import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
-import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+
 import org.apache.jackrabbit.oak.spi.commit.Observer;
 import org.apache.jackrabbit.oak.spi.mount.Mounts;
 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
 import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.apache.jackrabbit.oak.stats.Clock;
-import org.junit.After;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
-import javax.annotation.CheckForNull;
-import javax.annotation.Nonnull;
 import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 import static com.google.common.collect.ImmutableSet.of;
-import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
-import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
-import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
-import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
-import static 
org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
 import static org.junit.Assume.assumeTrue;
 
 @RunWith(Parameterized.class)
-public class ActiveDeletedBlobCollectionIT extends AbstractQueryTest {
-    private ExecutorService executorService = Executors.newFixedThreadPool(2);
+public class ActiveDeletedBlobCollectionIT extends 
AbstractActiveDeletedBlobTest {
 
     @Rule
-    public TemporaryFolder temporaryFolder = new TemporaryFolder(new 
File("target"));
-    @Rule
-    public TemporaryFolder blobCollectionRoot = new TemporaryFolder(new 
File("target"));
-    @Rule
-    public TemporaryFolder fileDataStoreRoot = new TemporaryFolder(new 
File("target"));
-    @Rule
     public MongoConnectionFactory connectionFactory = new 
MongoConnectionFactory();
 
     private MongoConnection mongoConnection = null;
-    private CountingBlobStore blobStore = null;
-
-    private Clock clock = new Clock.Virtual();
-    private ActiveDeletedBlobCollectorImpl adbc = null;
-
-    private AsyncIndexUpdate asyncIndexUpdate;
-
-    private LuceneIndexEditorProvider editorProvider;
-
-    private NodeStore nodeStore;
-
-    private LuceneIndexProvider provider;
 
     private final DataStoreType dataStoreType;
 
@@ -115,13 +68,6 @@ public class ActiveDeletedBlobCollection
         assumeTrue(MongoUtils.isAvailable());
     }
 
-    @After
-    public void after() {
-        new ExecutorCloser(executorService).close();
-        executorService.shutdown();
-        IndexDefinition.setDisableStoredIndexDefinition(false);
-    }
-
     enum DataStoreType {
         WITH_FDS,
         WITHOUT_FDS
@@ -140,11 +86,6 @@ public class ActiveDeletedBlobCollection
     }
 
     @Override
-    protected void createTestIndexNode() throws Exception {
-        setTraversalEnabled(false);
-    }
-
-    @Override
     protected ContentRepository createRepository() {
         adbc = new ActiveDeletedBlobCollectorImpl(clock,
                 new File(blobCollectionRoot.getRoot(), "deleted-blobs"), 
executorService);
@@ -182,14 +123,6 @@ public class ActiveDeletedBlobCollection
                 .createContentRepository();
     }
 
-    private IndexCopier createIndexCopier() {
-        try {
-            return new IndexCopier(executorService, temporaryFolder.getRoot());
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
     @Test
     public void simpleAsyncIndexUpdateBasedBlobCollection() throws Exception {
         createIndex("test1", of("propa"));
@@ -228,133 +161,4 @@ public class ActiveDeletedBlobCollection
         Assert.assertTrue("Second GC should delete some chunks too", 
secondGCNumChunks < firstGCNumChunks);
     }
 
-    private Tree createIndex(String name, Set<String> propNames) throws 
CommitFailedException {
-        Tree index = root.getTree("/");
-        return createIndex(index, name, propNames);
-    }
-
-    public static Tree createIndex(Tree index, String name, Set<String> 
propNames) throws CommitFailedException {
-        Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name);
-        def.setProperty(JcrConstants.JCR_PRIMARYTYPE,
-                INDEX_DEFINITIONS_NODE_TYPE, Type.NAME);
-        def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE);
-        def.setProperty(REINDEX_PROPERTY_NAME, true);
-        def.setProperty(ASYNC_PROPERTY_NAME, "async");
-        def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false);
-        
def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES,
 propNames, Type.STRINGS));
-        def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true);
-        return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name);
-    }
-
-    class CountingBlobStore implements GarbageCollectableBlobStore {
-
-        private final GarbageCollectableBlobStore delegate;
-        private long numChunks = 0;
-
-        CountingBlobStore(GarbageCollectableBlobStore delegate) {
-                this.delegate = delegate;
-        }
-
-        @Override
-        public String writeBlob(InputStream in) throws IOException {
-            String blobId = delegate.writeBlob(in);
-            numChunks += Iterators.size(delegate.resolveChunks(blobId));
-            return blobId;
-        }
-
-        @Override
-        public void setBlockSize(int x) {
-            delegate.setBlockSize(x);
-        }
-
-        @Override
-        public String writeBlob(InputStream in, BlobOptions options) throws 
IOException {
-            String blobId = delegate.writeBlob(in, options);
-            numChunks += Iterators.size(delegate.resolveChunks(blobId));
-            return blobId;
-        }
-
-        @Override
-        public String writeBlob(String tempFileName) throws IOException {
-            String blobId = delegate.writeBlob(tempFileName);
-            numChunks += Iterators.size(delegate.resolveChunks(blobId));
-            return blobId;
-        }
-
-        @Override
-        public int sweep() throws IOException {
-            return delegate.sweep();
-        }
-
-        @Override
-        public void startMark() throws IOException {
-            delegate.startMark();
-        }
-
-        @Override
-        public int readBlob(String blobId, long pos, byte[] buff, int off, int 
length) throws IOException {
-            return delegate.readBlob(blobId, pos, buff, off, length);
-        }
-
-        @Override
-        public void clearInUse() {
-            delegate.clearInUse();
-        }
-
-        @Override
-        public void clearCache() {
-            delegate.clearCache();
-        }
-
-        @Override
-        public long getBlobLength(String blobId) throws IOException {
-            return delegate.getBlobLength(blobId);
-        }
-
-        @Override
-        public long getBlockSizeMin() {
-            return delegate.getBlockSizeMin();
-        }
-
-        @Override
-        public Iterator<String> getAllChunkIds(long maxLastModifiedTime) 
throws Exception {
-            return delegate.getAllChunkIds(maxLastModifiedTime);
-        }
-
-        @Override
-        public InputStream getInputStream(String blobId) throws IOException {
-            return delegate.getInputStream(blobId);
-        }
-
-        @Override
-        @Deprecated
-        public boolean deleteChunks(List<String> chunkIds, long 
maxLastModifiedTime) throws Exception {
-            numChunks -= chunkIds.size();
-            return delegate.deleteChunks(chunkIds, maxLastModifiedTime);
-        }
-
-        @Override
-        @CheckForNull
-        public String getBlobId(@Nonnull String reference) {
-            return delegate.getBlobId(reference);
-        }
-
-        @Override
-        @CheckForNull
-        public String getReference(@Nonnull String blobId) {
-            return delegate.getReference(blobId);
-        }
-
-        @Override
-        public long countDeleteChunks(List<String> chunkIds, long 
maxLastModifiedTime) throws Exception {
-            long numDeleted = delegate.countDeleteChunks(chunkIds, 
maxLastModifiedTime);
-            numChunks -= numDeleted;
-            return numDeleted;
-        }
-
-        @Override
-        public Iterator<String> resolveChunks(String blobId) throws 
IOException {
-            return delegate.resolveChunks(blobId);
-        }
-    }
 }

Added: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java?rev=1804626&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
 Thu Aug 10 03:52:24 2017
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.plugins.blob.BlobTrackingStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.BlobIdTracker;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory
+    .ActiveDeletedBlobCollectorImpl;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static com.google.common.collect.ImmutableSet.of;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.intersection;
+import static com.google.common.collect.Sets.newHashSet;
+import static 
org.apache.jackrabbit.oak.spi.cluster.ClusterRepositoryInfo.getOrCreateId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ActiveDeletedBlobSyncTrackerTest extends 
AbstractActiveDeletedBlobTest {
+    @Rule
+    public TemporaryFolder blobTrackerRoot = new TemporaryFolder(new 
File("target"));
+
+    @Override
+    protected ContentRepository createRepository() {
+        try {
+            File blobCollectorDeleted = new File(blobCollectionRoot.getRoot(), 
"deleted-blobs");
+            blobCollectorDeleted.mkdirs();
+            adbc = new ActiveDeletedBlobCollectorImpl(clock, new 
File(blobCollectionRoot.getRoot(), "deleted-blobs"),
+                executorService);
+
+            IndexCopier copier = createIndexCopier();
+            editorProvider =
+                new LuceneIndexEditorProvider(copier, null, new 
ExtractedTextCache(10 * FileUtils.ONE_MB,
+                    100), null,
+                    Mounts.defaultMountInfoProvider(), adbc);
+            provider = new LuceneIndexProvider(copier);
+
+            OakFileDataStore ds = new OakFileDataStore();
+            ds.setMinRecordLength(10);
+            ds.init(fileDataStoreRoot.getRoot().getAbsolutePath());
+            DataStoreBlobStore dsbs = new DataStoreBlobStore(ds);
+            this.blobStore = new 
AbstractActiveDeletedBlobTest.CountingBlobStore(dsbs);
+
+            FileStore store = 
FileStoreBuilder.fileStoreBuilder(temporaryFolder.getRoot()).withMemoryMapping(false)
+                .withBlobStore(blobStore).build();
+            nodeStore = SegmentNodeStoreBuilders.builder(store).build();
+            BlobTrackingStore trackingStore = (BlobTrackingStore) blobStore;
+            trackingStore.addTracker(
+                new BlobIdTracker(blobTrackerRoot.getRoot().getAbsolutePath(), 
getOrCreateId(nodeStore), 600,
+                    dsbs));
+            // set the blob store to skip writing blobs through the node store
+            editorProvider.setBlobStore(blobStore);
+
+            asyncIndexUpdate = new AsyncIndexUpdate("async", nodeStore, 
editorProvider);
+            return new Oak(nodeStore).with(new InitialContent()).with(new 
OpenSecurityProvider())
+                .with((QueryIndexProvider) provider).with((Observer) 
provider).with(editorProvider)
+                .createContentRepository();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    // OAK-6504
+    @Test
+    public void syncActiveDeletionWithBlobTracker() throws Exception {
+        createIndex("test1", of("propa"));
+        root.getTree("/oak:index/counter").remove();
+        root.commit();
+        asyncIndexUpdate.run();
+        long initialNumChunks = blobStore.numChunks;
+
+        root.getTree("/").addChild("test").setProperty("propa", "foo");
+        root.commit();
+        asyncIndexUpdate.run();
+        long firstCommitNumChunks = blobStore.numChunks;
+        adbc.purgeBlobsDeleted(0, blobStore);//hack to purge file
+        long time = clock.getTimeIncreasing();
+        long hackPurgeNumChunks = blobStore.numChunks;
+        assertEquals("Hack purge must not purge any blob (first commit)",
+                firstCommitNumChunks, hackPurgeNumChunks);
+
+        root.getTree("/").addChild("test").setProperty("propa", "foo1");
+        root.commit();
+        asyncIndexUpdate.run();
+        long secondCommitNumChunks = blobStore.numChunks;
+        adbc.purgeBlobsDeleted(0, blobStore);//hack to purge file
+        hackPurgeNumChunks = blobStore.numChunks;
+        assertEquals("Hack purge must not purge any blob (second commit)",
+            secondCommitNumChunks, hackPurgeNumChunks);
+
+        List<String> beforeDeletionIds = 
newArrayList(blobStore.getAllChunkIds(-1));
+
+        adbc.purgeBlobsDeleted(time, blobStore);
+        adbc.getBlobDeletionCallback();
+        long firstGCNumChunks = blobStore.numChunks;
+
+        assertTrue("First commit must create some chunks", 
firstCommitNumChunks > initialNumChunks);
+        assertTrue("First commit must create some chunks", 
secondCommitNumChunks > firstCommitNumChunks);
+        assertTrue("First GC should delete some chunks", firstGCNumChunks < 
secondCommitNumChunks);
+
+        List<String> afterDeletionIds = 
newArrayList(blobStore.getAllChunkIds(-1));
+
+        assertTrackedDeleted(beforeDeletionIds, afterDeletionIds, blobStore);
+    }
+
+    private static void assertTrackedDeleted(List<String> beforeDeletionIds,
+        List<String> afterDeletionIds,
+        GarbageCollectableBlobStore blobStore) throws IOException {
+
+        // get the currently tracked ones
+        ArrayList<String> trackedIds = newArrayList(((BlobTrackingStore) 
blobStore).getTracker().get());
+        assertTrue("Tracked ids different from current blob list",
+            newHashSet(trackedIds).equals(newHashSet(afterDeletionIds)));
+
+        // get the deleted ones
+        beforeDeletionIds.removeAll(afterDeletionIds);
+        // tracked should not contain the deleted
+        assertTrue("Blob tracker not synced",
+            intersection(newHashSet(beforeDeletionIds), 
newHashSet(trackedIds)).isEmpty());
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
------------------------------------------------------------------------------
    svn:eol-style = native


Reply via email to