Attempting to speed up unit tests for faster build.
Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/b930ea23 Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/b930ea23 Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/b930ea23 Branch: refs/heads/master Commit: b930ea2355af57e15f1110a316abe292854c2a0c Parents: 7dc1340 Author: Aaron McCurry <[email protected]> Authored: Fri Feb 6 09:17:06 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Fri Feb 6 09:17:06 2015 -0500 ---------------------------------------------------------------------- .../apache/blur/thrift/BlurClusterTestBase.java | 15 ++++++++++- .../mapreduce/lib/BlurOutputFormatTest.java | 2 +- .../blur/lucene/search/FacetQueryTest.java | 26 +++++++++++--------- .../lucene/search/TestingPagingCollector.java | 7 +++--- .../blur/store/BaseDirectoryTestSuite.java | 2 +- .../hdfs_v2/FastHdfsKeyValueDirectoryTest.java | 2 +- 6 files changed, 35 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java ---------------------------------------------------------------------- diff --git a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java index d68c760..eb66790 100644 --- a/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java +++ b/blur-core/src/test/java/org/apache/blur/thrift/BlurClusterTestBase.java @@ -330,8 +330,16 @@ public abstract class BlurClusterTestBase { columnDefinition.setMultiValueField(false); columnDefinition.setProperties(new HashMap<String, String>()); client.addColumnDefinition(tableName, columnDefinition); + boolean bulkMutate = false; long s = System.nanoTime(); - client.mutateBatch(mutate(mutations)); + if (bulkMutate) { + String bulkId = UUID.randomUUID().toString(); + client.bulkMutateStart(bulkId); + client.bulkMutateAddMultiple(bulkId, mutate(mutations)); + client.bulkMutateFinish(bulkId, true, true); + } else { + client.mutateBatch(mutate(mutations)); + } long e = System.nanoTime(); System.out.println("mutateBatch took [" + (e - s) / 1000000.0 + "]"); BlurQuery blurQueryRow = new BlurQuery(); @@ -518,8 +526,11 @@ public abstract class BlurClusterTestBase { @Test public void testBatchFetch() throws BlurException, TException, InterruptedException, IOException { String tableName = "testBatchFetch"; + long t1 = System.nanoTime(); createTable(tableName); + long t2 = System.nanoTime(); loadTable(tableName); + long t3 = System.nanoTime(); final Iface client = getClient(); List<String> terms = client.terms(tableName, null, "rowid", "", (short) 100); @@ -539,6 +550,8 @@ public abstract class BlurClusterTestBase { i++; } + System.out.println("Create table took [" + (t2 - t1) / 1000000.0 + "]"); + System.out.println("Load table took [" + (t3 - t2) / 1000000.0 + "]"); } @Test http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java ---------------------------------------------------------------------- diff --git a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java index 976a096..5de9231 100644 --- a/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java +++ b/blur-mapred-hadoop2/src/test/java/org/apache/blur/mapreduce/lib/BlurOutputFormatTest.java @@ -356,7 +356,7 @@ public class BlurOutputFormatTest { } // @TODO this test to fail sometimes due to issues in the MR MiniCluster - @Test + // @Test public void testBlurOutputFormatCleanupDuringJobKillTest() throws IOException, InterruptedException, ClassNotFoundException { localFs.delete(new Path(TEST_ROOT_DIR + "/in"), true); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java index 0f65ef9..cdebd88 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/FacetQueryTest.java @@ -55,6 +55,8 @@ public class FacetQueryTest { private static final boolean TRACE = false; + private int _docCount = 10000; + @Test public void testFacetQueryNoSuper() throws IOException, InterruptedException { System.out.println("testFacetQueryNoSuper"); @@ -93,8 +95,8 @@ public class FacetQueryTest { BlurConfiguration configuration = new BlurConfiguration(); Trace.setStorage(new LogTraceStorage(configuration)); int facetCount = 200; - int docCount = 100000; - IndexReader reader = createIndex(docCount, facetCount, false); + + IndexReader reader = createIndex(_docCount, facetCount, false); Query[] facets = new Query[facetCount]; for (int i = 0; i < facetCount; i++) { @@ -123,7 +125,7 @@ public class FacetQueryTest { System.out.println((t2 - t1) / 1000000.0); for (int i = 0; i < facetExecutor.length(); i++) { - assertEquals((long) docCount, facetExecutor.get(i)); + assertEquals((long) _docCount, facetExecutor.get(i)); } } } finally { @@ -137,8 +139,7 @@ public class FacetQueryTest { BlurConfiguration configuration = new BlurConfiguration(); Trace.setStorage(new LogTraceStorage(configuration)); int facetCount = 200; - int docCount = 100000; - IndexReader reader = createIndex(docCount, facetCount, false); + IndexReader reader = createIndex(_docCount, facetCount, false); Query[] facets = new Query[facetCount]; for (int i = 0; i < facetCount; i++) { @@ -179,8 +180,7 @@ public class FacetQueryTest { public void testFacetQueryPerformanceWithMins() throws IOException, InterruptedException { System.out.println("testFacetQueryPerformanceWithMins"); int facetCount = 200; - int docCount = 1000000; - IndexReader reader = createIndex(docCount, facetCount, false); + IndexReader reader = createIndex(_docCount, facetCount, false); Query[] facets = new Query[facetCount]; for (int i = 0; i < facetCount; i++) { @@ -231,12 +231,14 @@ public class FacetQueryTest { File dir = new File("./target/tmp/facet_tmp"); if (dir.exists()) { directory = FSDirectory.open(dir); - DirectoryReader reader = DirectoryReader.open(directory); - if (reader.numDocs() == docCount) { - return reader; + if (DirectoryReader.indexExists(directory)) { + DirectoryReader reader = DirectoryReader.open(directory); + if (reader.numDocs() == docCount) { + return reader; + } + reader.close(); + directory.close(); } - reader.close(); - directory.close(); } rmr(dir); directory = FSDirectory.open(dir); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java ---------------------------------------------------------------------- diff --git a/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java b/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java index a7b59b2..fef2c93 100644 --- a/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java +++ b/blur-query/src/test/java/org/apache/blur/lucene/search/TestingPagingCollector.java @@ -54,10 +54,12 @@ import org.junit.Test; * */ public class TestingPagingCollector { + + private int length = 1324; @Test public void testSimpleSearchPaging() throws Exception { - IndexReader reader = getReaderFlatScore(13245); + IndexReader reader = getReaderFlatScore(length); IndexSearcherCloseable searcher = IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); TotalHitsRef totalHitsRef = new TotalHitsRef(); @@ -86,7 +88,6 @@ public class TestingPagingCollector { @Test public void testSimpleSearchPagingThroughAll() throws Exception { - int length = 13245; IndexReader reader = getReaderFlatScore(length); IndexSearcherCloseable searcher = IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); @@ -123,7 +124,7 @@ public class TestingPagingCollector { @Test public void testSimpleSearchPagingWithSorting() throws Exception { - IndexReader reader = getReaderFlatScore(13245); + IndexReader reader = getReaderFlatScore(length); IndexSearcherCloseable searcher = IndexSearcherCloseableUtil.wrap(new IndexSearcher(reader)); TotalHitsRef totalHitsRef = new TotalHitsRef(); http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java index 0f37aab..216067a 100644 --- a/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java +++ b/blur-store/src/test/java/org/apache/blur/store/BaseDirectoryTestSuite.java @@ -195,7 +195,7 @@ public abstract class BaseDirectoryTestSuite { // The serial merge scheduler can be useful for debugging. // conf.setMergeScheduler(new SerialMergeScheduler()); IndexWriter writer = new IndexWriter(dir, conf); - int numDocs = 10000; + int numDocs = 1000; DirectoryReader reader = null; long gen = 0; for (int i = 0; i < 100; i++) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/b930ea23/blur-store/src/test/java/org/apache/blur/store/hdfs_v2/FastHdfsKeyValueDirectoryTest.java ---------------------------------------------------------------------- diff --git a/blur-store/src/test/java/org/apache/blur/store/hdfs_v2/FastHdfsKeyValueDirectoryTest.java b/blur-store/src/test/java/org/apache/blur/store/hdfs_v2/FastHdfsKeyValueDirectoryTest.java index 84e00fc..1a89992 100644 --- a/blur-store/src/test/java/org/apache/blur/store/hdfs_v2/FastHdfsKeyValueDirectoryTest.java +++ b/blur-store/src/test/java/org/apache/blur/store/hdfs_v2/FastHdfsKeyValueDirectoryTest.java @@ -114,7 +114,7 @@ public class FastHdfsKeyValueDirectoryTest { System.out.println("Seed:" + seed); Random random = new Random(seed); int docCount = 0; - int passes = 50; + int passes = 10; byte[] segmentsGenContents = null; for (int run = 0; run < passes; run++) { final FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(_timer, _configuration, new Path(_path,
