Author: alexparvulescu Date: Wed Aug 12 14:10:38 2015 New Revision: 1695524
URL: http://svn.apache.org/r1695524 Log: OAK-2545 SegmentMk IT tests are too intensive - merged r1695521 Modified: jackrabbit/oak/branches/1.2/ (props changed) jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentOverflowExceptionIT.java jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java Propchange: jackrabbit/oak/branches/1.2/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Aug 12 14:10:38 2015 @@ -1,3 +1,3 @@ /jackrabbit/oak/branches/1.0:1665962 -/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820,1684868 ,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507 +/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820,1684868 ,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521 /jackrabbit/trunk:1345480 Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java Wed Aug 12 14:10:38 2015 @@ -69,7 +69,8 @@ import java.util.UUID; */ public class CompactionMap { - private final int compressInterval; + private static final int COMPRESS_INTERVAL = Integer.getInteger("compress-interval", 100000); + private final SegmentTracker tracker; private Map<RecordId, RecordId> recent = newHashMap(); @@ -87,8 +88,7 @@ public class CompactionMap { private CompactionMap prev; - CompactionMap(int compressInterval, SegmentTracker tracker) { - this.compressInterval = compressInterval; + CompactionMap(SegmentTracker tracker) { this.tracker = tracker; } @@ -208,7 +208,7 @@ public class CompactionMap { throw new IllegalArgumentException(); } recent.put(before, after); - if (recent.size() >= compressInterval) { + if (recent.size() >= COMPRESS_INTERVAL) { compress(); } } Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java Wed Aug 12 14:10:38 2015 @@ -83,7 +83,7 @@ public class Compactor { public Compactor(SegmentWriter writer, boolean cloneBinaries) { this.writer = writer; - this.map = new CompactionMap(100000, writer.getTracker()); + this.map = new CompactionMap(writer.getTracker()); this.cloneBinaries = cloneBinaries; } Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java Wed Aug 12 14:10:38 2015 @@ -104,7 +104,7 @@ public class SegmentTracker { this.writer = new SegmentWriter(store, this, version); this.cacheSize = cacheSizeMB * MB; this.compactionMap = new AtomicReference<CompactionMap>( - new CompactionMap(1, this)); + new CompactionMap(this)); } public SegmentTracker(SegmentStore store, SegmentVersion version) { Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java Wed Aug 12 14:10:38 2015 @@ -40,7 +40,7 @@ public class CompactionMapTest { System.out.println((runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024)); SegmentTracker factory = new MemoryStore().getTracker(); - CompactionMap map = new CompactionMap(100000, factory); + CompactionMap map = new CompactionMap(factory); for (int i = 0; i < 1000000; i++) { if (i % 1000 == 0) { System.gc(); @@ -59,13 +59,13 @@ public class CompactionMapTest { @Test public void testCompactionMap() { - int maxSegments = 1000; + int maxSegments = 100000; int maxEntriesPerSegment = 10; int seed = new Random().nextInt(); Random r = new Random(seed); SegmentTracker factory = new MemoryStore().getTracker(); - CompactionMap map = new CompactionMap(r.nextInt(maxSegments / 2), factory); + CompactionMap map = new CompactionMap(factory); Map<RecordId, RecordId> entries = newHashMap(); int segments = r.nextInt(maxSegments); Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentOverflowExceptionIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentOverflowExceptionIT.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentOverflowExceptionIT.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentOverflowExceptionIT.java Wed Aug 12 14:10:38 2015 @@ -21,9 +21,10 @@ package org.apache.jackrabbit.oak.plugin import static org.apache.commons.io.FileUtils.deleteDirectory; import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic; -import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD; import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.MEMORY_THRESHOLD_DEFAULT; +import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD; import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; +import static org.junit.Assume.assumeTrue; import java.io.ByteArrayInputStream; import java.io.File; @@ -33,7 +34,6 @@ import java.util.concurrent.Callable; import javax.annotation.Nonnull; -import com.google.common.collect.Iterables; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy; @@ -45,18 +45,29 @@ import org.apache.jackrabbit.oak.spi.sta import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.collect.Iterables; + /** - * Test for reproducing OAK-2662. This test will never terminate unless it fails, - * thus it is marked as @Ignored for now. + * <p>Tests verifying if the repository gets corrupted or not: {@code OAK-2662 SegmentOverflowException in HeavyWriteIT on Jenkins}</p> + * + * <p><b>This test will never terminate unless it fails</b>, thus it is disabled by default. On the + * command line specify {@code -DSegmentOverflowExceptionIT=true} to enable + * them.</p> + * + *<p>If you only want to run this test:<br> + * {@code mvn verify -Dsurefire.skip.ut=true -PintegrationTesting -Dit.test=SegmentOverflowExceptionIT -DSegmentOverflowExceptionIT=true} + * </p> */ -@Ignore("long running") public class SegmentOverflowExceptionIT { - private static final Logger LOG = LoggerFactory.getLogger(SegmentOverflowExceptionIT.class); + + private static final Logger LOG = LoggerFactory + .getLogger(SegmentOverflowExceptionIT.class); + private static final boolean ENABLED = Boolean + .getBoolean(SegmentOverflowExceptionIT.class.getSimpleName()); private final Random rnd = new Random(); @@ -64,6 +75,7 @@ public class SegmentOverflowExceptionIT @Before public void setUp() throws IOException { + assumeTrue(ENABLED); directory = File.createTempFile(getClass().getSimpleName(), "dir", new File("target")); directory.delete(); directory.mkdir(); @@ -72,7 +84,9 @@ public class SegmentOverflowExceptionIT @After public void cleanDir() { try { - deleteDirectory(directory); + if (directory != null) { + deleteDirectory(directory); + } } catch (IOException e) { LOG.error("Error cleaning directory", e); } Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/LargeNumberOfPropertiesTestIT.java Wed Aug 12 14:10:38 2015 @@ -18,31 +18,47 @@ */ package org.apache.jackrabbit.oak.plugins.segment.file; +import static org.apache.commons.io.FileUtils.deleteDirectory; +import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; +import static org.junit.Assume.assumeTrue; + import java.io.File; import java.io.IOException; -import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * <p>Tests verifying if the repository gets corrupted or not: {@code OAK-2481 IllegalStateException in TarMk with large number of properties}</p> + * + * <p>These tests are disabled by default due to their long running time. On the + * command line specify {@code -DLargeNumberOfPropertiesTestIT=true} to enable + * them.</p> + * + *<p>If you only want to run this test:<br> + * {@code mvn verify -Dsurefire.skip.ut=true -PintegrationTesting -Dit.test=LargeNumberOfPropertiesTestIT -DLargeNumberOfPropertiesTestIT=true} + * </p> + */ public class LargeNumberOfPropertiesTestIT { private static final Logger LOG = LoggerFactory - .getLogger(SegmentReferenceLimitTestIT.class); + .getLogger(LargeNumberOfPropertiesTestIT.class); + private static final boolean ENABLED = Boolean + .getBoolean(LargeNumberOfPropertiesTestIT.class.getSimpleName()); private File directory; @Before public void setUp() throws IOException { - directory = File.createTempFile("LargeNumberOfPropertiesTestIT", "dir", + assumeTrue(ENABLED); + directory = File.createTempFile(getClass().getSimpleName(), "dir", new File("target")); directory.delete(); directory.mkdir(); @@ -51,24 +67,18 @@ public class LargeNumberOfPropertiesTest @After public void cleanDir() { try { - FileUtils.deleteDirectory(directory); + if (directory != null) { + deleteDirectory(directory); + } } catch (IOException e) { LOG.error("Error cleaning directory", e); } } - /** - * OAK-2481 IllegalStateException in TarMk with large number of properties - * - * TODO Test is currently ignored because of how memory intensive it is - * - * @see <a - * href="https://issues.apache.org/jira/browse/OAK-2481";>OAK-2481</a> - */ @Test - @Ignore public void corruption() throws Exception { - FileStore fileStore = new FileStore(directory, 5, 0, false); + FileStore fileStore = newFileStore(directory).withMaxFileSize(5) + .withNoCache().withMemoryMapping(true).create(); SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore); NodeBuilder root = nodeStore.getRoot().builder(); @@ -78,7 +88,7 @@ public class LargeNumberOfPropertiesTest // i=26 hits the hard limit for the number of properties a node can // have (262144) for (int i = 0; i < 25; i++) { - System.out.println(i); + LOG.debug("run {}/24", i); for (int j = 0; j < 10000; j++) { c.setProperty("int-" + i + "-" + j, i); } Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java?rev=1695524&r1=1695523&r2=1695524&view=diff ============================================================================== --- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java (original) +++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/SegmentReferenceLimitTestIT.java Wed Aug 12 14:10:38 2015 @@ -19,6 +19,10 @@ package org.apache.jackrabbit.oak.plugins.segment.file; +import static org.apache.commons.io.FileUtils.deleteDirectory; +import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore; +import static org.junit.Assume.assumeTrue; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -27,7 +31,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; -import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; @@ -36,20 +39,35 @@ import org.apache.jackrabbit.oak.spi.sta import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * <p>Tests verifying if the repository gets corrupted or not: {@code OAK-2294 Corrupt repository after concurrent version operations}</p> + * + * <p>These tests are disabled by default due to their long running time. On the + * command line specify {@code -DSegmentReferenceLimitTestIT=true} to enable + * them.</p> + * + *<p>If you only want to run this test:<br> + * {@code mvn verify -Dsurefire.skip.ut=true -PintegrationTesting -Dit.test=SegmentReferenceLimitTestIT -DSegmentReferenceLimitTestIT=true} + * </p> + */ public class SegmentReferenceLimitTestIT { - private static final Logger LOG = LoggerFactory.getLogger(SegmentReferenceLimitTestIT.class); + private static final Logger LOG = LoggerFactory + .getLogger(SegmentReferenceLimitTestIT.class); + private static final boolean ENABLED = Boolean + .getBoolean(SegmentReferenceLimitTestIT.class.getSimpleName()); private File directory; @Before public void setUp() throws IOException { - directory = File.createTempFile("SegmentReferenceLimitTestIT", "dir", new File("target")); + assumeTrue(ENABLED); + directory = File.createTempFile(getClass().getSimpleName(), "dir", + new File("target")); directory.delete(); directory.mkdir(); } @@ -57,24 +75,18 @@ public class SegmentReferenceLimitTestIT @After public void cleanDir() { try { - FileUtils.deleteDirectory(directory); + if (directory != null) { + deleteDirectory(directory); + } } catch (IOException e) { LOG.error("Error cleaning directory", e); } } - /** - * OAK-2294 Corrupt repository after concurrent version operations - * - * TODO Test is currently ignored because of how memory intensive it is - * - * @see <a - * href="https://issues.apache.org/jira/browse/OAK-2294";>OAK-2294</a> - */ @Test - @Ignore public void corruption() throws IOException, CommitFailedException, ExecutionException, InterruptedException { - FileStore fileStore = new FileStore(directory, 1, 0, false); + FileStore fileStore = newFileStore(directory).withMaxFileSize(1) + .withNoCache().withMemoryMapping(true).create(); SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore); NodeBuilder root = nodeStore.getRoot().builder();
