This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch 1.9 in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 2c1c72f5cf4375b8afdd3662783760d8ab62e5bf Author: Christopher Tubbs <[email protected]> AuthorDate: Fri Jan 31 07:47:26 2020 -0500 Fix #1482 Increase timeouts on some slow ITs Stabilize some ITs by increasing timeouts to values which cause them to pass more frequently on modest developer hardware and adding a timeout to one that was missing it. Also backport the removal of some flakey performance-related tests, which were already removed in newer branches. --- .../org/apache/accumulo/test/ExistingMacIT.java | 2 +- .../test/RecoveryCompactionsAreFlushesIT.java | 2 +- .../org/apache/accumulo/test/WaitForBalanceIT.java | 2 +- .../accumulo/test/functional/ConstraintIT.java | 2 +- .../test/functional/MetadataMaxFilesIT.java | 2 +- .../accumulo/test/master/SuspendedTabletsIT.java | 9 +- .../test/performance/RollWALPerformanceIT.java | 135 --------------------- .../performance/metadata/FastBulkImportIT.java | 113 ----------------- .../accumulo/test/proxy/ProxyDurabilityIT.java | 2 +- .../replication/MultiInstanceReplicationIT.java | 2 +- 10 files changed, 14 insertions(+), 257 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java index da22fa8..83815ec 100644 --- a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java @@ -59,7 +59,7 @@ import org.junit.Test; public class ExistingMacIT extends ConfigurableMacBase { @Override public int defaultTimeoutSeconds() { - return 2 * 60; + return 4 * 60; } @Override diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java index 4e11274..255b228 100644 --- a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java @@ -46,7 +46,7 @@ public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterHarness { @Override public int defaultTimeoutSeconds() { - return 60; + return 180; } @Override diff --git a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java index 45e307d..a1625bc 100644 --- a/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java +++ b/test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java @@ -43,7 +43,7 @@ public class WaitForBalanceIT extends ConfigurableMacBase { @Override public int defaultTimeoutSeconds() { - return 60; + return 120; } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java b/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java index d0b2540..5d1b511 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ConstraintIT.java @@ -51,7 +51,7 @@ public class ConstraintIT extends AccumuloClusterHarness { @Override protected int defaultTimeoutSeconds() { - return 30; + return 60; } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java index b4b04d8..1cc9aaf 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java @@ -56,7 +56,7 @@ public class MetadataMaxFilesIT extends ConfigurableMacBase { @Override protected int defaultTimeoutSeconds() { - return 4 * 60; + return 5 * 60; } @Test diff --git a/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java b/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java index c86ecf0..b1009df 100644 --- a/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java +++ b/test/src/main/java/org/apache/accumulo/test/master/SuspendedTabletsIT.java @@ -84,6 +84,11 @@ public class SuspendedTabletsIT extends ConfigurableMacBase { public static final int TABLETS = 100; @Override + protected int defaultTimeoutSeconds() { + return 5 * 60; + } + + @Override public void configure(MiniAccumuloConfigImpl cfg, Configuration fsConf) { cfg.setProperty(Property.TABLE_SUSPEND_DURATION, SUSPEND_DURATION + "ms"); cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s"); @@ -273,8 +278,8 @@ public class SuspendedTabletsIT extends ConfigurableMacBase { assertTrue(recoverTime - killTime >= NANOSECONDS.convert(SUSPEND_DURATION, MILLISECONDS)); } - private static interface TServerKiller { - public void eliminateTabletServers(ClientContext ctx, TabletLocations locs, int count) + private interface TServerKiller { + void eliminateTabletServers(ClientContext ctx, TabletLocations locs, int count) throws Exception; } diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java deleted file mode 100644 index 8c5d1ab..0000000 --- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.performance; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; - -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.metadata.MetadataTable; -import org.apache.accumulo.core.metadata.RootTable; -import org.apache.accumulo.minicluster.ServerType; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.accumulo.test.categories.MiniClusterOnlyTests; -import org.apache.accumulo.test.categories.PerformanceTests; -import org.apache.accumulo.test.continuous.ContinuousIngest; -import org.apache.accumulo.test.functional.ConfigurableMacBase; -import org.apache.accumulo.test.mrit.IntegrationTestMapReduce; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.Text; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category({MiniClusterOnlyTests.class, PerformanceTests.class}) -public class RollWALPerformanceIT extends ConfigurableMacBase { - - @BeforeClass - static public void checkMR() { - assumeFalse(IntegrationTestMapReduce.isMapReduce()); - } - - @Override - protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1"); - cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M"); - cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100"); - cfg.setProperty(Property.GC_FILE_ARCHIVE, "false"); - cfg.setProperty(Property.GC_CYCLE_START, "1s"); - cfg.setProperty(Property.GC_CYCLE_DELAY, "1s"); - cfg.useMiniDFS(true); - } - - @Override - protected int defaultTimeoutSeconds() { - return 20 * 60; - } - - private long ingest() throws Exception { - final Connector c = getConnector(); - final String tableName = getUniqueNames(1)[0]; - - log.info("Creating the table"); - c.tableOperations().create(tableName); - - log.info("Splitting the table"); - final long SPLIT_COUNT = 100; - final long distance = Long.MAX_VALUE / SPLIT_COUNT; - final SortedSet<Text> splits = new TreeSet<>(); - for (int i = 1; i < SPLIT_COUNT; i++) { - splits.add(new Text(String.format("%016x", i * distance))); - } - c.tableOperations().addSplits(tableName, splits); - - log.info("Waiting for balance"); - c.instanceOperations().waitForBalance(); - - final Instance inst = c.getInstance(); - - log.info("Starting ingest"); - final long start = System.currentTimeMillis(); - final String args[] = {"-i", inst.getInstanceName(), "-z", inst.getZooKeepers(), "-u", "root", - "-p", ROOT_PASSWORD, "--batchThreads", "2", "--table", tableName, "--num", - Long.toString(50 * 1000), // 50K 100 byte entries - }; - - ContinuousIngest.main(args); - final long result = System.currentTimeMillis() - start; - log.debug(String.format("Finished in %,d ms", result)); - log.debug("Dropping table"); - c.tableOperations().delete(tableName); - return result; - } - - private long getAverage() throws Exception { - final int REPEAT = 3; - long totalTime = 0; - for (int i = 0; i < REPEAT; i++) { - totalTime += ingest(); - } - return totalTime / REPEAT; - } - - private void testWalPerformanceOnce() throws Exception { - // get time with a small WAL, which will cause many WAL roll-overs - long avg1 = getAverage(); - // use a bigger WAL max size to eliminate WAL roll-overs - Connector c = getConnector(); - c.instanceOperations().setProperty(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1G"); - c.tableOperations().flush(MetadataTable.NAME, null, null, true); - c.tableOperations().flush(RootTable.NAME, null, null, true); - getCluster().getClusterControl().stop(ServerType.TABLET_SERVER); - getCluster().start(); - long avg2 = getAverage(); - log.info(String.format("Average run time with small WAL %,d with large WAL %,d", avg1, avg2)); - assertTrue(avg1 > avg2); - double percent = (100. * avg1) / avg2; - log.info(String.format("Percent of large log: %.2f%%", percent)); - assertTrue(percent < 125.); - } - - @Test - public void testWalPerformance() throws Exception { - testWalPerformanceOnce(); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java b/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java deleted file mode 100644 index 4bd301c..0000000 --- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/FastBulkImportIT.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.test.performance.metadata; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeFalse; - -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.file.FileOperations; -import org.apache.accumulo.core.file.FileSKVWriter; -import org.apache.accumulo.core.file.rfile.RFile; -import org.apache.accumulo.core.util.CachedConfiguration; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.accumulo.test.functional.ConfigurableMacBase; -import org.apache.accumulo.test.mrit.IntegrationTestMapReduce; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.junit.BeforeClass; -import org.junit.Test; - -// ACCUMULO-3327 -public class FastBulkImportIT extends ConfigurableMacBase { - - @BeforeClass - static public void checkMR() { - assumeFalse(IntegrationTestMapReduce.isMapReduce()); - } - - @Override - protected int defaultTimeoutSeconds() { - return 120; - } - - @Override - protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - cfg.setNumTservers(3); - cfg.setProperty(Property.TSERV_BULK_ASSIGNMENT_THREADS, "5"); - cfg.setProperty(Property.TSERV_BULK_PROCESS_THREADS, "5"); - cfg.setProperty(Property.TABLE_MAJC_RATIO, "9999"); - cfg.setProperty(Property.TABLE_FILE_MAX, "9999"); - cfg.setProperty(Property.TABLE_DURABILITY, "none"); - } - - @Test - public void test() throws Exception { - log.info("Creating table"); - final String tableName = getUniqueNames(1)[0]; - final Connector c = getConnector(); - c.tableOperations().create(tableName); - log.info("Adding splits"); - SortedSet<Text> splits = new TreeSet<>(); - for (int i = 1; i < 0xfff; i += 7) { - splits.add(new Text(Integer.toHexString(i))); - } - c.tableOperations().addSplits(tableName, splits); - - log.info("Creating lots of bulk import files"); - FileSystem fs = getCluster().getFileSystem(); - Path basePath = getCluster().getTemporaryPath(); - CachedConfiguration.setInstance(fs.getConf()); - - Path base = new Path(basePath, "testBulkFail_" + tableName); - fs.delete(base, true); - fs.mkdirs(base); - Path bulkFailures = new Path(base, "failures"); - Path files = new Path(base, "files"); - fs.mkdirs(bulkFailures); - fs.mkdirs(files); - for (int i = 0; i < 100; i++) { - FileSKVWriter writer = FileOperations.getInstance().newWriterBuilder() - .forFile(files.toString() + "/bulk_" + i + "." + RFile.EXTENSION, fs, fs.getConf()) - .withTableConfiguration(AccumuloConfiguration.getDefaultConfiguration()).build(); - writer.startDefaultLocalityGroup(); - for (int j = 0x100; j < 0xfff; j += 3) { - writer.append(new Key(Integer.toHexString(j)), new Value(new byte[0])); - } - writer.close(); - } - log.info("Waiting for balance"); - c.instanceOperations().waitForBalance(); - - log.info("Bulk importing files"); - long now = System.currentTimeMillis(); - c.tableOperations().importDirectory(tableName, files.toString(), bulkFailures.toString(), true); - double diffSeconds = (System.currentTimeMillis() - now) / 1000.; - log.info(String.format("Import took %.2f seconds", diffSeconds)); - assertTrue(diffSeconds < 30); - } - -} diff --git a/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java b/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java index e1d5291..11e940b 100644 --- a/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java +++ b/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java @@ -63,7 +63,7 @@ public class ProxyDurabilityIT extends ConfigurableMacBase { @Override protected int defaultTimeoutSeconds() { - return 120; + return 240; } @Override diff --git a/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java b/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java index ddf115a..aea9dd1 100644 --- a/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java @@ -84,7 +84,7 @@ public class MultiInstanceReplicationIT extends ConfigurableMacBase { @Override public int defaultTimeoutSeconds() { - return 10 * 60; + return 15 * 60; } @Before
