Author: mduerig
Date: Fri Aug 11 15:01:35 2017
New Revision: 1804793

URL: http://svn.apache.org/viewvc?rev=1804793&view=rev
Log:
OAK-6399: Re-establish full offline compaction functionality
Remove binary dedup feature

Modified:
    
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
    
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java?rev=1804793&r1=1804792&r2=1804793&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
 Fri Aug 11 15:01:35 2017
@@ -105,13 +105,6 @@ public class SegmentGCOptions {
 
     private int memoryThreshold = MEMORY_THRESHOLD_DEFAULT;
 
-    private boolean ocBinDeduplication = Boolean
-            .getBoolean("oak.segment.compaction.binaryDeduplication");
-
-    private long ocBinMaxSize = Long.getLong(
-            "oak.segment.compaction.binaryDeduplicationMaxSize",
-            100 * 1024 * 1024);
-
     private long gcSizeDeltaEstimation = Long.getLong(
             "oak.segment.compaction.gcSizeDeltaEstimation",
             SIZE_DELTA_ESTIMATION_DEFAULT);
@@ -247,9 +240,7 @@ public class SegmentGCOptions {
         if (offline) {
             return getClass().getSimpleName() + "{" +
                     "offline=" + offline +
-                    ", retainedGenerations=" + retainedGenerations +
-                    ", ocBinDeduplication=" + ocBinDeduplication +
-                    ", ocBinMaxSize=" + ocBinMaxSize + "}";
+                    ", retainedGenerations=" + retainedGenerations + "}";
         } else {
             return getClass().getSimpleName() + "{" +
                     "paused=" + paused +
@@ -291,37 +282,6 @@ public class SegmentGCOptions {
         return this;
     }
 
-    /**
-     * Offline compaction only. Enables content based de-duplication of
-     * binaries. Involves a fair amount of I/O when reading/comparing
-     * potentially equal blobs. set via the
-     * 'oak.segment.compaction.binaryDeduplication' system property
-     * @return this instance.
-     */
-    public SegmentGCOptions withBinaryDeduplication() {
-        this.ocBinDeduplication = true;
-        return this;
-    }
-
-    public boolean isBinaryDeduplication() {
-        return this.ocBinDeduplication;
-    }
-
-    /**
-     * Offline compaction only. Set the upper bound for the content based
-     * de-duplication checks.
-     * @param binMaxSize
-     * @return this instance
-     */
-    public SegmentGCOptions setBinaryDeduplicationMaxSize(long binMaxSize) {
-        this.ocBinMaxSize = binMaxSize;
-        return this;
-    }
-
-    public long getBinaryDeduplicationMaxSize() {
-        return this.ocBinMaxSize;
-    }
-
     public long getGcSizeDeltaEstimation() {
         return gcSizeDeltaEstimation;
     }

Modified: 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1804793&r1=1804792&r2=1804793&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
 Fri Aug 11 15:01:35 2017
@@ -84,7 +84,6 @@ import org.apache.jackrabbit.oak.stats.C
 import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
 import org.apache.jackrabbit.oak.stats.StatisticsProvider;
 import org.junit.BeforeClass;
-import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -370,118 +369,6 @@ public class CompactionAndCleanupIT {
         }
     }
 
-    @Test
-    @Ignore("OAK-6399") // FIXME OAK-6399: this test fails because the content 
based binary deduplication feature is missing in Compactor
-    public void offlineCompactionBinC1() throws Exception {
-        SegmentGCOptions gcOptions = defaultGCOptions().setOffline()
-                .withBinaryDeduplication();
-        ScheduledExecutorService executor = newSingleThreadScheduledExecutor();
-        FileStore fileStore = fileStoreBuilder(getFileStoreFolder())
-                .withMaxFileSize(1)
-                .withGCOptions(gcOptions)
-                .withStatisticsProvider(new 
DefaultStatisticsProvider(executor))
-                .build();
-        SegmentNodeStore nodeStore = SegmentNodeStoreBuilders
-                .builder(fileStore).build();
-
-        try {
-            NodeBuilder extra = nodeStore.getRoot().builder();
-            NodeBuilder content = extra.child("content");
-
-            int blobSize = 5 * 1024 * 1024;
-            byte[] data = new byte[blobSize];
-            new Random().nextBytes(data);
-
-            NodeBuilder c1 = content.child("c1");
-            Blob b1 = nodeStore.createBlob(new ByteArrayInputStream(data));
-            c1.setProperty("blob1", b1);
-            NodeBuilder c2 = content.child("c2");
-            Blob b2 = nodeStore.createBlob(new ByteArrayInputStream(data));
-            c2.setProperty("blob2", b2);
-            nodeStore.merge(extra, EmptyHook.INSTANCE, CommitInfo.EMPTY);
-            fileStore.flush();
-
-            int cpNo = 4;
-            Set<String> cps = new HashSet<String>();
-            for (int i = 0; i < cpNo; i++) {
-                cps.add(nodeStore.checkpoint(60000));
-            }
-            assertEquals(cpNo, cps.size());
-            for (String cp : cps) {
-                assertTrue(nodeStore.retrieve(cp) != null);
-            }
-
-            long size1 = fileStore.getStats().getApproximateSize();
-            fileStore.compactFull();
-            fileStore.cleanup();
-            long size2 = fileStore.getStats().getApproximateSize();
-            assertSize("with compacted binaries", size2, 0, size1 - blobSize);
-        } finally {
-            fileStore.close();
-        }
-    }
-
-    /**
-     * Create 2 binary nodes with same content but not same reference. Reduce
-     * the max size if de-duplicated binaries under the binary length. Verify
-     * de-duplication capabilities of compaction.
-     */
-    @Test
-    public void offlineCompactionBinC2() throws Exception {
-        int blobSize = 5 * 1024 * 1024;
-
-        SegmentGCOptions gcOptions = defaultGCOptions().setOffline()
-                .withBinaryDeduplication()
-                .setBinaryDeduplicationMaxSize(blobSize / 2);
-        ScheduledExecutorService executor = newSingleThreadScheduledExecutor();
-        FileStore fileStore = fileStoreBuilder(getFileStoreFolder())
-                .withMaxFileSize(1)
-                .withGCOptions(gcOptions)
-                .withStatisticsProvider(new 
DefaultStatisticsProvider(executor))
-                .build();
-        SegmentNodeStore nodeStore = SegmentNodeStoreBuilders
-                .builder(fileStore).build();
-
-        try {
-            NodeBuilder extra = nodeStore.getRoot().builder();
-            NodeBuilder content = extra.child("content");
-
-            byte[] data = new byte[blobSize];
-            new Random().nextBytes(data);
-
-            NodeBuilder c1 = content.child("c1");
-            Blob b1 = nodeStore.createBlob(new ByteArrayInputStream(data));
-            c1.setProperty("blob1", b1);
-            NodeBuilder c2 = content.child("c2");
-            Blob b2 = nodeStore.createBlob(new ByteArrayInputStream(data));
-            c2.setProperty("blob2", b2);
-            nodeStore.merge(extra, EmptyHook.INSTANCE, CommitInfo.EMPTY);
-            fileStore.flush();
-
-            int cpNo = 4;
-            Set<String> cps = new HashSet<String>();
-            for (int i = 0; i < cpNo; i++) {
-                cps.add(nodeStore.checkpoint(60000));
-            }
-            assertEquals(cpNo, cps.size());
-            for (String cp : cps) {
-                assertTrue(nodeStore.retrieve(cp) != null);
-            }
-
-            long size1 = fileStore.getStats().getApproximateSize();
-            fileStore.compactFull();
-            fileStore.cleanup();
-            long size2 = fileStore.getStats().getApproximateSize();
-
-            // not expected to reduce the size too much, as the binaries are
-            // above the threshold
-            assertSize("with compacted binaries", size2, size1 * 9 / 10,
-                    size1 * 11 / 10);
-        } finally {
-            fileStore.close();
-        }
-    }
-
     /**
      * Create 2 binary nodes with same content and same reference. Verify
      * de-duplication capabilities of compaction


Reply via email to