Repository: geode
Updated Branches:
refs/heads/develop 68af1df8d -> 96bc0ebf1
GEODE-3128: Changed the batch size to 1000 when creating AEQFactory
This closes #603
Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/96bc0ebf
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/96bc0ebf
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/96bc0ebf
Branch: refs/heads/develop
Commit: 96bc0ebf1cc5123bab2b743d3a8ae4c282d86d06
Parents: 68af1df
Author: David Anuta <[email protected]>
Authored: Mon Jun 19 08:55:34 2017 -0700
Committer: nabarun <[email protected]>
Committed: Wed Jul 12 14:50:12 2017 -0700
----------------------------------------------------------------------
.../cache/lucene/internal/LuceneIndexImpl.java | 1 +
.../lucene/internal/filesystem/FileSystem.java | 1 +
.../lucene/LuceneQueriesIntegrationTest.java | 17 ++-
.../cache/lucene/MixedObjectIndexDUnitTest.java | 20 +--
.../IndexRepositoryImplPerformanceTest.java | 58 +++++----
.../LuceneIndexMemoryOverheadTest.java | 129 +++++++++++++++++++
6 files changed, 190 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
index 3393bcf..31efd8a 100644
---
a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
+++
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/LuceneIndexImpl.java
@@ -175,6 +175,7 @@ public abstract class LuceneIndexImpl implements
InternalLuceneIndex {
}
factory.setMaximumQueueMemory(1000);
factory.setDispatcherThreads(10);
+ factory.setBatchSize(1000);
factory.setIsMetaQueue(true);
if (attributes.getDataPolicy().withPersistence()) {
factory.setPersistent(true);
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
index 164955f..3facef0 100644
---
a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
+++
b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/filesystem/FileSystem.java
@@ -69,6 +69,7 @@ public class FileSystem {
throw new IOException("File exists.");
}
stats.incFileCreates(1);
+
// TODO unlock region ?
return file;
}
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
index 685d61c..3c0b10d 100644
---
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
+++
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/LuceneQueriesIntegrationTest.java
@@ -14,8 +14,13 @@
*/
package org.apache.geode.cache.lucene;
-import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.*;
-import static org.junit.Assert.*;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.DEFAULT_FIELD;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.IntRangeQueryProvider;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.verifyQueryKeyAndValues;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.verifyQueryKeys;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.HashMap;
@@ -26,9 +31,6 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
-import org.apache.geode.cache.CacheLoader;
-import org.apache.geode.cache.CacheLoaderException;
-import org.apache.geode.cache.LoaderHelper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -44,6 +46,9 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.ExpectedException;
+import org.apache.geode.cache.CacheLoader;
+import org.apache.geode.cache.CacheLoaderException;
+import org.apache.geode.cache.LoaderHelper;
import org.apache.geode.cache.Region;
import org.apache.geode.cache.RegionShortcut;
import org.apache.geode.cache.lucene.test.TestObject;
@@ -59,7 +64,7 @@ public class LuceneQueriesIntegrationTest extends
LuceneIntegrationTest {
@Rule
public ExpectedException thrown = ExpectedException.none();
private static final String INDEX_NAME = "index";
- protected static final String REGION_NAME = "index";
+ public static final String REGION_NAME = "index";
private Region region;
@Test()
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
index 949ce25..90bf44a 100644
---
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
+++
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/MixedObjectIndexDUnitTest.java
@@ -15,22 +15,26 @@
package org.apache.geode.cache.lucene;
-import static org.apache.geode.cache.lucene.test.LuceneTestUtilities.*;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.FloatRangeQueryProvider;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.IntRangeQueryProvider;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.REGION_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import org.apache.geode.cache.Region;
-import org.apache.geode.test.dunit.SerializableRunnableIF;
-import org.apache.geode.test.junit.categories.DistributedTest;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.runner.RunWith;
-
import java.io.Serializable;
import java.util.List;
import java.util.stream.IntStream;
+
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.test.dunit.SerializableRunnableIF;
+import org.apache.geode.test.junit.categories.DistributedTest;
@Category(DistributedTest.class)
@RunWith(JUnitParamsRunner.class)
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
index 46425b9..4b321a5 100644
---
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
+++
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/IndexRepositoryImplPerformanceTest.java
@@ -15,20 +15,20 @@
package org.apache.geode.cache.lucene.internal.repository;
-import org.apache.geode.DataSerializable;
-import org.apache.geode.DataSerializer;
-import org.apache.geode.cache.*;
-import org.apache.geode.cache.asyncqueue.AsyncEventQueue;
-import org.apache.geode.cache.lucene.*;
-import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
-import org.apache.geode.cache.lucene.internal.LuceneServiceImpl;
-import org.apache.geode.cache.lucene.internal.directory.RegionDirectory;
-import org.apache.geode.cache.lucene.internal.distributed.TopEntriesCollector;
-import org.apache.geode.cache.lucene.internal.filesystem.ChunkKey;
-import org.apache.geode.cache.lucene.internal.filesystem.File;
-import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
-import
org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
-import org.apache.geode.test.junit.categories.PerformanceTest;
+import static org.apache.geode.distributed.ConfigurationProperties.LOG_LEVEL;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
@@ -45,14 +45,28 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.geode.distributed.ConfigurationProperties.*;
+import org.apache.geode.DataSerializable;
+import org.apache.geode.DataSerializer;
+import org.apache.geode.cache.Cache;
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.cache.PartitionAttributesFactory;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.cache.asyncqueue.AsyncEventQueue;
+import org.apache.geode.cache.lucene.LuceneIndex;
+import org.apache.geode.cache.lucene.LuceneQuery;
+import org.apache.geode.cache.lucene.LuceneQueryException;
+import org.apache.geode.cache.lucene.LuceneQueryProvider;
+import org.apache.geode.cache.lucene.LuceneService;
+import org.apache.geode.cache.lucene.LuceneServiceProvider;
+import org.apache.geode.cache.lucene.PageableLuceneQueryResults;
+import org.apache.geode.cache.lucene.internal.LuceneIndexStats;
+import org.apache.geode.cache.lucene.internal.LuceneServiceImpl;
+import org.apache.geode.cache.lucene.internal.directory.RegionDirectory;
+import org.apache.geode.cache.lucene.internal.distributed.TopEntriesCollector;
+import org.apache.geode.cache.lucene.internal.filesystem.FileSystemStats;
+import
org.apache.geode.cache.lucene.internal.repository.serializer.HeterogeneousLuceneSerializer;
+import org.apache.geode.test.junit.categories.PerformanceTest;
/**
http://git-wip-us.apache.org/repos/asf/geode/blob/96bc0ebf/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
----------------------------------------------------------------------
diff --git
a/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
new file mode 100644
index 0000000..28047ce
--- /dev/null
+++
b/geode-lucene/src/test/java/org/apache/geode/cache/lucene/internal/repository/LuceneIndexMemoryOverheadTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
contributor license
+ * agreements. See the NOTICE file distributed with this work for additional
information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache
License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express
+ * or implied. See the License for the specific language governing permissions
and limitations under
+ * the License.
+ */
+package org.apache.geode.cache.lucene.internal.repository;
+
+
+import static
org.apache.geode.cache.lucene.LuceneQueriesIntegrationTest.REGION_NAME;
+import static
org.apache.geode.cache.lucene.test.LuceneTestUtilities.INDEX_NAME;
+
+import java.nio.charset.Charset;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.logging.log4j.Logger;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.springframework.shell.converters.AvailableCommandsConverter;
+
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.cache.lucene.LuceneIntegrationTest;
+import org.apache.geode.cache.lucene.test.TestObject;
+import org.apache.geode.cache.partition.PartitionRegionHelper;
+import org.apache.geode.internal.cache.InternalCache;
+import org.apache.geode.internal.cache.TombstoneService;
+import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.internal.size.ObjectGraphSizer;
+import org.apache.geode.test.junit.categories.PerformanceTest;
+
+@Category(PerformanceTest.class)
+@Ignore("Tests have no assertions")
+public class LuceneIndexMemoryOverheadTest extends LuceneIntegrationTest {
+ private static int NUM_BATCHES = 30;
+ private static int NUM_ENTRIES = 10000;
+ private Random random = new Random(0);
+
+ private static final Logger logger = LogService.getLogger();
+ private Callable flush =
+ () -> luceneService.waitUntilFlushed(INDEX_NAME, REGION_NAME, 60000,
TimeUnit.MILLISECONDS);
+ protected static ObjectGraphSizer.ObjectFilter filter =
+ (parent, object) -> !(object instanceof AvailableCommandsConverter);
+
+ @Test
+ public void perEntryNonIndexMemoryWithTombstoneGC() throws Exception {
+ measureOverhead(((InternalCache) cache).getTombstoneService(), () -> true);
+ }
+
+ @Test
+ public void perEntryNonIndexMemoryWithoutTombstoneGC() throws Exception {
+ measureOverhead(null, () -> true);
+ }
+
+ @Test
+ public void perEntryIndexMemoryWithTombstoneGC() throws Exception {
+ luceneService.createIndexFactory().setFields("field1",
"field2").create(INDEX_NAME,
+ REGION_NAME);
+
+ measureOverhead(((InternalCache) cache).getTombstoneService(), flush);
+ }
+
+ @Test
+ public void perEntryIndexMemoryWithoutTombstoneGC() throws Exception {
+ luceneService.createIndexFactory().setFields("field1",
"field2").create(INDEX_NAME,
+ REGION_NAME);
+
+ measureOverhead(null, flush);
+ }
+
+ private void waitForFlush(Callable flush) throws Exception {
+ while (flush.call().equals(false)) {
+ logger.info("Flush returned false. Waiting for flush to return true...");
+ }
+ }
+
+ private void measureOverhead(TombstoneService service, Callable flush)
throws Exception {
+
+ Region region =
cache.createRegionFactory(RegionShortcut.PARTITION).create(REGION_NAME);
+ PartitionRegionHelper.assignBucketsToPartitions(region);
+
+ long emptySize = ObjectGraphSizer.size(cache, filter, false);
+ waitForFlush(flush);
+
+ long size;
+ int numEntries = 0;
+
+ for (int batch = 0; batch < NUM_BATCHES; batch++) {
+
+ size =
+ addEntriesAndGetMemorySize(region, numEntries, NUM_ENTRIES, service,
flush) - emptySize;
+ numEntries = region.size();
+
+ logger.info(
+ numEntries + " entries take up " + size + ", per entry overhead " +
size / numEntries);
+ }
+ }
+
+ private long addEntriesAndGetMemorySize(Region region, int start, int count,
+ TombstoneService service, Callable flush) throws Exception {
+ int c = 0;
+ for (int i = start; i < start + count; i++) {
+ byte[] field1 = new byte[3];
+ byte[] field2 = new byte[5];
+ random.nextBytes(field1);
+ random.nextBytes(field2);
+ TestObject test = new TestObject(new String(field1,
Charset.forName("ASCII")),
+ new String(field2, Charset.forName("ASCII")));
+ region.put(i, test);
+ }
+
+ waitForFlush(flush);
+ if (service != null)
+ service.forceBatchExpirationForTests((int)
service.getScheduledTombstoneCount());
+
+ return ObjectGraphSizer.size(cache, filter, false);
+ }
+}
