GEODE-11: Fixing bug that arises while using user defined analyzer in gfsh.
This closes #233 When we use an user defined analyzer class in gfsh, the cluster hangs because the create region message that is sent has a LuceneIndexCreationProfile has a user defined analyzer class. Fixed this by storing only the class name in LuceneIndexCreationProfile. Signed-off-by: Gester Zhou <[email protected]> GEODE-11: StandardAnalyzer is used when analyzer option is null Using the string "null" in the analyzer option for create lucene index command will now set the field analyzer to be StandardAnalyzer. GEODE-11: Added CliAvailabilityIndicator to lucene gfsh commands The lucene gfsh commands were available offline. Changed this behavior by adding CliAvailabilityIndicator. GEODE-11:Fix for "Comparison method violates general contract" The search results are sorted by score and the compareTo method caused the "Comparison method violates general contract" error when it returned multiple results with same score. Signed-off-by: Gester Zhou <[email protected]> GEODE 11: Added a check to make sure StandardAnalyzer and NULL were considered the same. LuceneIndexCreationProfile checks to makes sure the newly created index's field analyzers are consistent with existing indexes' field analyzers. However, when a field's analyzer is null and a new index is created with StandardAnalyzer on the same field, it failed. Fixed this issue by modifying the conditional checks. Added dunit tests to verify. Signed-off-by: Gester Zhou <[email protected]> GEODE 11: Added a check to make sure StandardAnalyzer and NULL were considered the same. LuceneIndexCreationProfile checks to makes sure the newly created index's field analyzers are consistent with existing indexes' field analyzers. However, when a field's analyzer is null and a new index is created with StandardAnalyzer on the same field, it failed. Fixed this issue by modifying the conditional checks. Added dunit tests to verify. Signed-off-by: Gester Zhou <[email protected]> GEODE 11: Clean up of comparison code in LuceneIndexCreationProfile Since the null in fieldAnalyzers are converted to Standard analyzer, a few checks become irrelevant. Modified the code and relevant tests. Signed-off-by: Gester Zhou <[email protected]> fix merge issue Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/1ab59ffb Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/1ab59ffb Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/1ab59ffb Branch: refs/heads/feature/GEODE-420 Commit: 1ab59ffb20346a957cfcbc8324376f772f14da96 Parents: cea5535 Author: Aparna Dharmakkan <[email protected]> Authored: Thu Aug 11 16:35:41 2016 -0700 Committer: zhouxh <[email protected]> Committed: Fri Aug 19 11:46:39 2016 -0700 ---------------------------------------------------------------------- .../internal/LuceneIndexCreationProfile.java | 98 +++++++------------- .../internal/cli/LuceneIndexCommands.java | 7 ++ .../lucene/internal/cli/LuceneIndexDetails.java | 8 +- .../internal/cli/LuceneSearchResults.java | 2 +- .../functions/LuceneCreateIndexFunction.java | 12 ++- .../lucene/LuceneIndexCreationDUnitTest.java | 58 +++++++++--- .../LuceneIndexCreationProfileJUnitTest.java | 11 ++- .../cli/LuceneIndexCommandsDUnitTest.java | 34 +++++++ .../cli/LuceneIndexCommandsJUnitTest.java | 40 ++++++++ .../cache/lucene/test/LuceneTestUtilities.java | 11 +-- 10 files changed, 186 insertions(+), 95 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfile.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfile.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfile.java index 12f4ae5..720d20d 100644 --- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfile.java +++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfile.java @@ -22,6 +22,7 @@ import com.gemstone.gemfire.internal.Version; import com.gemstone.gemfire.internal.cache.CacheServiceProfile; import com.gemstone.gemfire.internal.i18n.LocalizedStrings; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; import java.io.DataInput; import java.io.DataOutput; @@ -34,9 +35,9 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri private String[] fieldNames; - private Class<? extends Analyzer> analyzerClass; + private String analyzerClass; - private Map<String, Class<? extends Analyzer>> fieldAnalyzers; + private Map<String, String> fieldAnalyzers; private String regionPath; @@ -47,7 +48,7 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri Map<String, Analyzer> fieldAnalyzers) { this.indexName = indexName; this.fieldNames = fieldNames; - this.analyzerClass = analyzer.getClass(); + this.analyzerClass = analyzer.getClass().getSimpleName(); initializeFieldAnalyzers(fieldAnalyzers); } @@ -56,7 +57,7 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri this.indexName = indexName; this.regionPath = regionPath; this.fieldNames = fieldNames; - this.analyzerClass = analyzer.getClass(); + this.analyzerClass = analyzer.getClass().getSimpleName(); initializeFieldAnalyzers(fieldAnalyzers); } @@ -68,20 +69,21 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri return this.fieldNames; } - public Class<? extends Analyzer> getAnalyzerClass() { + public String getAnalyzerClass() { return this.analyzerClass; } - public Map<String, Class<? extends Analyzer>> getFieldAnalyzers() { + public Map<String, String> getFieldAnalyzers() { return this.fieldAnalyzers; } protected void initializeFieldAnalyzers(Map<String, Analyzer> fieldAnalyzers) { - if (fieldAnalyzers != null && !fieldAnalyzers.isEmpty()) { - this.fieldAnalyzers = new HashMap<>(); - for (Map.Entry<String, Analyzer> entry : fieldAnalyzers.entrySet()) { - // Null values are allowed in analyzers which means the default Analyzer is used - this.fieldAnalyzers.put(entry.getKey(), entry.getValue() == null ? null : entry.getValue().getClass()); + this.fieldAnalyzers = new HashMap<>(); + for (String field : fieldNames) { + if(fieldAnalyzers != null && !fieldAnalyzers.isEmpty()) { + this.fieldAnalyzers.put(field, fieldAnalyzers.get(field) == null ? StandardAnalyzer.class.getSimpleName() : fieldAnalyzers.get(field).getClass().getSimpleName()); + } else { + this.fieldAnalyzers.put(field, StandardAnalyzer.class.getSimpleName()); } } } @@ -94,15 +96,14 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri @Override public String checkCompatibility(String regionPath, CacheServiceProfile profile) { String result = null; - LuceneIndexCreationProfile myProfile = (LuceneIndexCreationProfile) profile; - if (myProfile == null) { + LuceneIndexCreationProfile remoteProfile = (LuceneIndexCreationProfile) profile; + if (remoteProfile == null) { // TODO This can occur if one member defines no indexes but another one does. Currently this is caught by the async event id checks. } else { // Verify fields are the same - if (!Arrays.equals(myProfile.getFieldNames(), getFieldNames())) { - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_FIELDS_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_FIELDS_3 - .toString(myProfile.getIndexName(), regionPath, Arrays.toString(getFieldNames()), - Arrays.toString(myProfile.getFieldNames())); + if (!Arrays.equals(remoteProfile.getFieldNames(), getFieldNames())) { + return LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_FIELDS_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_FIELDS_3 + .toString(getIndexName(), regionPath, Arrays.toString(getFieldNames()), Arrays.toString(remoteProfile.getFieldNames())); } // Verify the analyzer class is the same @@ -116,49 +117,22 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri } */ - // Verify the field analyzer fields and classes are the same if either member sets field analyzers - if (myProfile.getFieldAnalyzers() != null || getFieldAnalyzers() != null) { - // Check for one member defining field analyzers while the other member does not - if (myProfile.getFieldAnalyzers() == null) { - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_FIELD_ANALYZERS_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_NO_FIELD_ANALYZERS - .toString(myProfile.getIndexName(), regionPath, getFieldAnalyzers()); - } else if (getFieldAnalyzers() == null) { - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_NO_FIELD_ANALYZERS_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_FIELD_ANALYZERS_2 - .toString(myProfile.getIndexName(), regionPath, myProfile.getFieldAnalyzers()); - } else { - // Both local and remote analyzers are set. Verify the sizes of the field analyzers are identical - if (myProfile.getFieldAnalyzers().size() != getFieldAnalyzers().size()) { - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_FIELD_ANALYZERS_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_FIELD_ANALYZERS_3 - .toString(myProfile.getIndexName(), regionPath, getFieldAnalyzers(), - myProfile.getFieldAnalyzers()); - } - - // Iterate the existing analyzers and compare them to the input analyzers - // Note: This is currently destructive to the input field analyzers map which should be ok since its a transient object. - for (Iterator<Map.Entry<String, Class<? extends Analyzer>>> i = myProfile.getFieldAnalyzers().entrySet().iterator(); i.hasNext(); ) { - Map.Entry<String, Class<? extends Analyzer>> entry = i.next(); - // Remove the existing field's analyzer from the input analyzers - Class<? extends Analyzer> analyzerClass = getFieldAnalyzers().remove(entry.getKey()); - - // Verify the input field analyzer matches the current analyzer - if (analyzerClass == null && entry.getValue() != null) { - // The input field analyzers do not include the existing field analyzer - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_NO_ANALYZER_ON_FIELD_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_ANALYZER_3_ON_THAT_FIELD - .toString(myProfile.getIndexName(), regionPath, entry.getKey(), entry.getValue().getName()); - break; - } else if (analyzerClass != null && entry.getValue() == null) { - // The existing field analyzers do not include the input field analyzer - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_ANALYZER_2_ON_FIELD_3_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_NO_ANALYZER_ON_THAT_FIELD - .toString(myProfile.getIndexName(), regionPath, analyzerClass.getName(), entry.getKey()); - break; - } else { - if (analyzerClass != entry.getValue()) { - // The class of the input analyzer does not match the existing analyzer for the field - result = LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_ANALYZER_2_ON_FIELD_3_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_ANALYZER_4_ON_THAT_FIELD - .toString(myProfile.getIndexName(), regionPath, analyzerClass.getName(), entry.getKey(), entry.getValue().getName()); - break; - } - } + // Iterate the existing analyzers and compare them to the input analyzers + // Note: This is currently destructive to the input field analyzers map which should be ok since its a transient object. + if (!getFieldAnalyzers().equals(remoteProfile.getFieldAnalyzers())) { + if (getFieldAnalyzers().size() != remoteProfile.getFieldAnalyzers().size()) { + return LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_FIELDS_2_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_FIELDS_3 + .toString(getIndexName(), regionPath, + Arrays.toString(getFieldAnalyzers().keySet().toArray()), + Arrays.toString(remoteProfile.getFieldAnalyzers().keySet().toArray())); + } + // now the 2 maps should have the same size + for (String field:getFieldAnalyzers().keySet()) { + if (!remoteProfile.getFieldAnalyzers().get(field).equals(getFieldAnalyzers().get(field))) { + return LocalizedStrings.LuceneService_CANNOT_CREATE_INDEX_0_ON_REGION_1_WITH_ANALYZER_2_ON_FIELD_3_BECAUSE_ANOTHER_MEMBER_DEFINES_THE_SAME_INDEX_WITH_ANALYZER_4_ON_THAT_FIELD + .toString(getIndexName(), regionPath, + getFieldAnalyzers().get(field), field, + remoteProfile.getFieldAnalyzers().get(field)); } } } @@ -170,7 +144,7 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri public void toData(DataOutput out) throws IOException { DataSerializer.writeString(this.indexName, out); DataSerializer.writeStringArray(this.fieldNames, out); - DataSerializer.writeClass(this.analyzerClass, out); + DataSerializer.writeString(this.analyzerClass, out); DataSerializer.writeHashMap(this.fieldAnalyzers, out); } @@ -178,7 +152,7 @@ public class LuceneIndexCreationProfile implements CacheServiceProfile, DataSeri public void fromData(DataInput in) throws IOException, ClassNotFoundException { this.indexName = DataSerializer.readString(in); this.fieldNames = DataSerializer.readStringArray(in); - this.analyzerClass = (Class<? extends Analyzer>) DataSerializer.readClass(in); + this.analyzerClass = DataSerializer.readString(in); this.fieldAnalyzers = DataSerializer.readHashMap(in); } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommands.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommands.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommands.java index 20c3990..200a14f 100755 --- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommands.java +++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommands.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import org.apache.geode.security.ResourcePermission.Operation; import org.apache.geode.security.ResourcePermission.Resource; +import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; @@ -460,4 +461,10 @@ public class LuceneIndexCommands extends AbstractCommandsSupport { throw new IllegalArgumentException("Region not found."); return CliUtil.executeFunction(searchIndexFunction, queryInfo, targetMembers); } + + @CliAvailabilityIndicator({LuceneCliStrings.LUCENE_SEARCH_INDEX, LuceneCliStrings.LUCENE_CREATE_INDEX, + LuceneCliStrings.LUCENE_DESCRIBE_INDEX, LuceneCliStrings.LUCENE_LIST_INDEX}) + public boolean indexCommandsAvailable() { + return (!CliUtil.isGfshVM() || (getGfsh() != null && getGfsh().isConnectedAndReady())); + } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexDetails.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexDetails.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexDetails.java index 5d9c5c6..eaec4a0 100644 --- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexDetails.java +++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexDetails.java @@ -89,7 +89,7 @@ public class LuceneIndexDetails implements Comparable<LuceneIndexDetails>, Seria return results; } - private Map<String, String> getFieldAnalyzerStringsFromProfile(Map<String, Class <? extends Analyzer>> fieldAnalyzers) { + private Map<String, String> getFieldAnalyzerStringsFromProfile(Map<String, String> fieldAnalyzers) { if(fieldAnalyzers == null) { return Collections.emptyMap(); @@ -97,10 +97,10 @@ public class LuceneIndexDetails implements Comparable<LuceneIndexDetails>, Seria Map<String, String> results = new HashMap<>(); - for (Entry<String, Class<? extends Analyzer>> entry : fieldAnalyzers.entrySet()) { - final Class analyzer = entry.getValue(); + for (Entry<String, String> entry : fieldAnalyzers.entrySet()) { + final String analyzer = entry.getValue(); if(analyzer != null) { - results.put(entry.getKey(), analyzer.getSimpleName()); + results.put(entry.getKey(), analyzer); } } return results; http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneSearchResults.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneSearchResults.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneSearchResults.java index 7f07366..9f03873 100644 --- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneSearchResults.java +++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneSearchResults.java @@ -57,7 +57,7 @@ public class LuceneSearchResults<K,V> implements Comparable<LuceneSearchResults> @Override public int compareTo(final LuceneSearchResults searchResults) { - return getScore() < searchResults.getScore() ? -1 : 1; + return Float.compare(getScore(),searchResults.getScore()); } public boolean getExeptionFlag() { return exceptionFlag; } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java index d943fbc..cb45f7b 100644 --- a/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java +++ b/geode-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java @@ -26,20 +26,20 @@ import com.gemstone.gemfire.cache.Cache; import com.gemstone.gemfire.cache.CacheFactory; import com.gemstone.gemfire.cache.execute.FunctionAdapter; import com.gemstone.gemfire.cache.execute.FunctionContext; -import com.gemstone.gemfire.cache.lucene.LuceneIndex; import com.gemstone.gemfire.cache.lucene.LuceneService; import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; -import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneCliStrings; import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexDetails; import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexInfo; import com.gemstone.gemfire.internal.InternalEntity; +import com.gemstone.gemfire.internal.lang.StringUtils; import com.gemstone.gemfire.management.internal.cli.CliUtil; import com.gemstone.gemfire.management.internal.cli.functions.CliFunctionResult; import com.gemstone.gemfire.management.internal.cli.i18n.CliStrings; import com.gemstone.gemfire.management.internal.configuration.domain.XmlEntity; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; /** @@ -78,6 +78,7 @@ public class LuceneCreateIndexFunction extends FunctionAdapter implements Intern service.createIndex(indexInfo.getIndexName(), indexInfo.getRegionPath(), fields); } else { + if (analyzerName.length != fields.length) throw new Exception("Mismatch in lengths of fields and analyzers"); Map<String, Analyzer> fieldAnalyzer = new HashMap<>(); for (int i = 0; i < fields.length; i++) { Analyzer analyzer = toAnalyzer(analyzerName[i]); @@ -97,8 +98,13 @@ public class LuceneCreateIndexFunction extends FunctionAdapter implements Intern } } - private Analyzer toAnalyzer(final String className) + private Analyzer toAnalyzer(String className) { + if (className==null) + className=StandardAnalyzer.class.getCanonicalName(); + else if (StringUtils.trim(className).equals("") | StringUtils.trim(className).equals("null") ) + className = StandardAnalyzer.class.getCanonicalName(); + Class<? extends Analyzer> clazz = CliUtil.forName(className, LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER); return CliUtil.newInstance(clazz, LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER); } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneIndexCreationDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneIndexCreationDUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneIndexCreationDUnitTest.java index cec8d0e..e8abb4c 100644 --- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneIndexCreationDUnitTest.java +++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/LuceneIndexCreationDUnitTest.java @@ -19,6 +19,7 @@ package com.gemstone.gemfire.cache.lucene; import com.gemstone.gemfire.cache.RegionShortcut; +import com.gemstone.gemfire.cache.lucene.test.LuceneTestUtilities; import com.gemstone.gemfire.test.dunit.SerializableRunnableIF; import com.gemstone.gemfire.test.junit.categories.DistributedTest; import com.gemstone.gemfire.util.test.TestUtil; @@ -125,7 +126,7 @@ public class LuceneIndexCreationDUnitTest extends LuceneDUnitTest { dataStore1.invoke(() -> initDataStore(createIndex1)); SerializableRunnableIF createIndex2 = getAnalyzersIndexWithOneField(); - dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD2)); + dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS_2)); } @Test @@ -134,7 +135,7 @@ public class LuceneIndexCreationDUnitTest extends LuceneDUnitTest { dataStore1.invoke(() -> initDataStore(createIndex1)); SerializableRunnableIF createIndex2 = getAnalyzersIndexWithTwoFields(); - dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZER_SIZES_1)); + dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS)); } @Test @@ -143,25 +144,26 @@ public class LuceneIndexCreationDUnitTest extends LuceneDUnitTest { dataStore1.invoke(() -> initDataStore(createIndex1)); SerializableRunnableIF createIndex2 = getAnalyzersIndexWithOneField(KeywordAnalyzer.class); - dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS)); + dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2)); } @Test public void verifyDifferentFieldAnalyzersFails2() { - SerializableRunnableIF createIndex1 = getAnalyzersIndexWithNullField2(); + SerializableRunnableIF createIndex1 = getAnalyzersIndexWithNullField1(); dataStore1.invoke(() -> initDataStore(createIndex1)); - SerializableRunnableIF createIndex2 = getAnalyzersIndexWithNullField1(); - dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD1)); + SerializableRunnableIF createIndex2 = getAnalyzersIndexWithNullField2(); + dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS)); } @Test public void verifyDifferentFieldAnalyzersFails3() { - SerializableRunnableIF createIndex1 = getAnalyzersIndexWithNullField1(); + SerializableRunnableIF createIndex1 = getAnalyzersIndexWithNullField2(); dataStore1.invoke(() -> initDataStore(createIndex1)); - SerializableRunnableIF createIndex2 = getAnalyzersIndexWithNullField2(); - dataStore2.invoke(() -> initDataStore(createIndex2, CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_EXISTING_MEMBER)); + SerializableRunnableIF createIndex2 = getAnalyzersIndexWithNullField1(); + dataStore2.invoke(() -> initDataStore(createIndex2, + LuceneTestUtilities.CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_3)); } @Test @@ -258,18 +260,36 @@ public class LuceneIndexCreationDUnitTest extends LuceneDUnitTest { private final Object[] getXmlAndExceptionMessages() { return $( new Object[] { "verifyDifferentFieldsFails", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS }, - new Object[] { "verifyDifferentFieldAnalyzerSizesFails1", CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD2 }, - new Object[] { "verifyDifferentFieldAnalyzerSizesFails2", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZER_SIZES_1 }, - new Object[] { "verifyDifferentFieldAnalyzersFails1", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS }, + new Object[] { "verifyDifferentFieldAnalyzerSizesFails1", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS_2 }, + new Object[] { "verifyDifferentFieldAnalyzerSizesFails2", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS }, + new Object[] { "verifyDifferentFieldAnalyzersFails1", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2 }, // Currently setting a null analyzer is not a valid xml configuration: <lucene:field name="field2" analyzer="null"/> - //new Object[] { "verifyDifferentFieldAnalyzersFails2", CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD1 }, - //new Object[] { "verifyDifferentFieldAnalyzersFails3", CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_EXISTING_MEMBER }, + //new Object[] { "verifyDifferentFieldAnalyzersFails2", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_1 }, + //new Object[] { "verifyDifferentFieldAnalyzersFails3", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2 }, new Object[] { "verifyDifferentIndexNamesFails", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_NAMES }, new Object[] { "verifyDifferentIndexesFails1", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_INDEXES_1 }, new Object[] { "verifyDifferentIndexesFails2", CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_INDEXES_2 } ); } + @Test + public void verifyStandardAnalyzerAndNullOnSameFieldPasses() { + SerializableRunnableIF createIndex1 = getAnalyzersIndexWithNullField1(); + dataStore1.invoke(() -> initDataStore(createIndex1)); + + SerializableRunnableIF createIndex2 = getAnalyzersIndexWithTwoFields2(); + dataStore2.invoke(() -> initDataStore(createIndex2)); + } + + @Test + public void verifyStandardAnalyzerAndNullOnSameFieldPasses2() { + SerializableRunnableIF createIndex1 = getAnalyzersIndexWithTwoFields2(); + dataStore1.invoke(() -> initDataStore(createIndex1)); + + SerializableRunnableIF createIndex2 = getAnalyzersIndexWithNullField1(); + dataStore2.invoke(() -> initDataStore(createIndex2)); + } + private String getXmlFileForTest(String testName) { return TestUtil.getResourcePath(getClass(), getClass().getSimpleName() + "." + testName + ".cache.xml"); } @@ -388,4 +408,14 @@ public class LuceneIndexCreationDUnitTest extends LuceneDUnitTest { luceneService.createIndex(INDEX_NAME, REGION_NAME, analyzers); }; } + + private SerializableRunnableIF getAnalyzersIndexWithTwoFields2() { + return () -> { + LuceneService luceneService = LuceneServiceProvider.get(getCache()); + Map<String, Analyzer> analyzers = new HashMap<>(); + analyzers.put("field1", new StandardAnalyzer()); + analyzers.put("field2", new KeywordAnalyzer()); + luceneService.createIndex(INDEX_NAME, REGION_NAME, analyzers); + }; + } } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfileJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfileJUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfileJUnitTest.java index ecb48af..c64dd3d 100644 --- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfileJUnitTest.java +++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/LuceneIndexCreationProfileJUnitTest.java @@ -19,6 +19,7 @@ package com.gemstone.gemfire.cache.lucene.internal; import com.gemstone.gemfire.CopyHelper; +import com.gemstone.gemfire.cache.lucene.test.LuceneTestUtilities; import com.gemstone.gemfire.test.junit.categories.UnitTest; import junitparams.JUnitParamsRunner; import junitparams.Parameters; @@ -78,27 +79,27 @@ public class LuceneIndexCreationProfileJUnitTest { new Object[] { getTwoAnalyzersLuceneIndexCreationProfile(), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), - CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD2 + CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS }, new Object[] { getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), getTwoAnalyzersLuceneIndexCreationProfile(), - CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZER_SIZES_2 + CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_3 }, new Object[] { getOneAnalyzerLuceneIndexCreationProfile(new StandardAnalyzer()), getOneAnalyzerLuceneIndexCreationProfile(new KeywordAnalyzer()), - CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS + CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2 }, new Object[] { getNullField2AnalyzerLuceneIndexCreationProfile(), getNullField1AnalyzerLuceneIndexCreationProfile(), - CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD1 + CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_1 }, new Object[] { getNullField1AnalyzerLuceneIndexCreationProfile(), getNullField2AnalyzerLuceneIndexCreationProfile(), - CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_EXISTING_MEMBER + LuceneTestUtilities.CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2 } ); } http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsDUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsDUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsDUnitTest.java index ce7c7b2..1ead454 100755 --- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsDUnitTest.java +++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsDUnitTest.java @@ -26,6 +26,7 @@ import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexCreationProfile; import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; import com.gemstone.gemfire.cache.lucene.internal.LuceneServiceImpl; import com.gemstone.gemfire.distributed.ConfigurationProperties; +import com.gemstone.gemfire.internal.lang.StringUtils; import com.gemstone.gemfire.management.cli.Result.Status; import com.gemstone.gemfire.management.internal.cli.CommandManager; import com.gemstone.gemfire.management.internal.cli.commands.CliCommandTestBase; @@ -45,9 +46,12 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; + import static com.gemstone.gemfire.cache.lucene.test.LuceneTestUtilities.*; import static com.gemstone.gemfire.test.dunit.Assert.*; +import static junitparams.JUnitParamsRunner.$; import java.io.Serializable; import java.util.ArrayList; @@ -62,7 +66,11 @@ import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; + @Category(DistributedTest.class) +@RunWith(JUnitParamsRunner.class) public class LuceneIndexCommandsDUnitTest extends CliCommandTestBase { @@ -267,6 +275,32 @@ public class LuceneIndexCommandsDUnitTest extends CliCommandTestBase { } @Test + public void createIndexWithNullAnalyzerShouldUseStandardAnalyzer() throws Exception { + final VM vm1 = Host.getHost(0).getVM(1); + vm1.invoke(() -> {getCache();}); + + CommandManager.getInstance().add(LuceneIndexCommands.class.newInstance()); + String analyzerList=StandardAnalyzer.class.getCanonicalName()+",null,"+KeywordAnalyzer.class.getCanonicalName(); + CommandStringBuilder csb = new CommandStringBuilder(LuceneCliStrings.LUCENE_CREATE_INDEX); + csb.addOption(LuceneCliStrings.LUCENE__INDEX_NAME,INDEX_NAME); + csb.addOption(LuceneCliStrings.LUCENE__REGION_PATH,REGION_NAME); + csb.addOption(LuceneCliStrings.LUCENE_CREATE_INDEX__FIELD,"field1,field2,field3"); + csb.addOption(LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER,analyzerList); + + String resultAsString = executeCommandAndLogResult(csb); + + vm1.invoke(() -> { + LuceneService luceneService = LuceneServiceProvider.get(getCache()); + createRegion(); + final LuceneIndex index = luceneService.getIndex(INDEX_NAME, REGION_NAME); + final Map<String, Analyzer> fieldAnalyzers = index.getFieldAnalyzers(); + assertEquals(StandardAnalyzer.class.getCanonicalName(), fieldAnalyzers.get("field1").getClass().getCanonicalName()); + assertEquals(StandardAnalyzer.class.getCanonicalName(), fieldAnalyzers.get("field2").getClass().getCanonicalName()); + assertEquals(KeywordAnalyzer.class.getCanonicalName(), fieldAnalyzers.get("field3").getClass().getCanonicalName()); + }); + } + + @Test public void describeIndexShouldReturnExistingIndex() throws Exception { final VM vm1 = Host.getHost(0).getVM(1); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsJUnitTest.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsJUnitTest.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsJUnitTest.java index 81f0025..71cf3d3 100644 --- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsJUnitTest.java +++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/cli/LuceneIndexCommandsJUnitTest.java @@ -324,6 +324,46 @@ public class LuceneIndexCommandsJUnitTest { assertEquals(Arrays.asList("C","B","A"), data.retrieveAllValues("key")); } + @Test + public void testSearchIndexWhenSearchResultsHaveSameScore() throws Exception { + + final Cache mockCache = mock(Cache.class, "Cache"); + final ResultCollector mockResultCollector = mock(ResultCollector.class, "ResultCollector"); + final LuceneIndexCommands commands=spy(createIndexCommands(mockCache,null)); + + final List<Set<LuceneSearchResults>> queryResultsList = new ArrayList<>(); + HashSet<LuceneSearchResults> queryResults = new HashSet<>(); + queryResults.add(createQueryResults("A","Result1",1)); + queryResults.add(createQueryResults("B","Result1",1)); + queryResults.add(createQueryResults("C","Result1",1)); + queryResults.add(createQueryResults("D","Result1",1)); + queryResults.add(createQueryResults("E","Result1",1)); + queryResults.add(createQueryResults("F","Result1",1)); + queryResults.add(createQueryResults("G","Result1",1)); + queryResults.add(createQueryResults("H","Result1",1)); + queryResults.add(createQueryResults("I","Result1",1)); + queryResults.add(createQueryResults("J","Result1",1)); + queryResults.add(createQueryResults("K","Result1",1)); + queryResults.add(createQueryResults("L","Result1",1)); + queryResults.add(createQueryResults("M","Result1",1)); + queryResults.add(createQueryResults("N","Result1",1)); + queryResults.add(createQueryResults("P","Result1",1)); + queryResults.add(createQueryResults("Q","Result1",1)); + queryResults.add(createQueryResults("R","Result1",1)); + queryResults.add(createQueryResults("S","Result1",1)); + queryResults.add(createQueryResults("T","Result1",1)); + queryResultsList.add(queryResults); + + doReturn(mockResultCollector).when(commands).executeSearch(isA(LuceneQueryInfo.class)); + doReturn(queryResultsList).when(mockResultCollector).getResult(); + + CommandResult result = (CommandResult) commands.searchIndex("index","region","Result1","field1",-1,-1, true); + + TabularResultData data = (TabularResultData) result.getResultData(); + + assertEquals(queryResults.size(), data.retrieveAllValues("key").size()); + } + private String getPage(final LuceneSearchResults[] expectedResults, int[] indexList) { final TabularResultData data = ResultBuilder.createTabularResultData(); for (int i:indexList) { http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1ab59ffb/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/test/LuceneTestUtilities.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/test/LuceneTestUtilities.java b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/test/LuceneTestUtilities.java index da281ff..3fae251 100644 --- a/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/test/LuceneTestUtilities.java +++ b/geode-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/test/LuceneTestUtilities.java @@ -48,12 +48,11 @@ public class LuceneTestUtilities { public static final String DEFAULT_FIELD = "text"; public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS = "Cannot create Lucene index index on region /region with fields [field1, field2] because another member defines the same index with fields [field1]."; - public static final String CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD2 = "Cannot create Lucene index index on region /region with no analyzer on field field2 because another member defines the same index with analyzer org.apache.lucene.analysis.core.KeywordAnalyzer on that field."; - public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZER_SIZES_1 = "Cannot create Lucene index index on region /region with field analyzers {field2=class org.apache.lucene.analysis.core.KeywordAnalyzer, field1=class org.apache.lucene.analysis.core.KeywordAnalyzer} because another member defines the same index with field analyzers {field1=class org.apache.lucene.analysis.core.KeywordAnalyzer}."; - public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZER_SIZES_2 = "Cannot create Lucene index index on region /region with field analyzers {field1=class org.apache.lucene.analysis.core.KeywordAnalyzer, field2=class org.apache.lucene.analysis.core.KeywordAnalyzer} because another member defines the same index with field analyzers {field1=class org.apache.lucene.analysis.core.KeywordAnalyzer}."; - public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS = "Cannot create Lucene index index on region /region with analyzer org.apache.lucene.analysis.core.KeywordAnalyzer on field field1 because another member defines the same index with analyzer org.apache.lucene.analysis.standard.StandardAnalyzer on that field."; - public static final String CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_FIELD1 = "Cannot create Lucene index index on region /region with no analyzer on field field1 because another member defines the same index with analyzer org.apache.lucene.analysis.core.KeywordAnalyzer on that field."; - public static final String CANNOT_CREATE_LUCENE_INDEX_NO_ANALYZER_EXISTING_MEMBER = "Cannot create Lucene index index on region /region with analyzer org.apache.lucene.analysis.core.KeywordAnalyzer on field field1 because another member defines the same index with no analyzer on that field."; + public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_FIELDS_2 = "Cannot create Lucene index index on region /region with fields [field1] because another member defines the same index with fields [field1, field2]."; + public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS = "Cannot create Lucene index index on region /region with analyzer StandardAnalyzer on field field2 because another member defines the same index with analyzer KeywordAnalyzer on that field."; + public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_1 = "Cannot create Lucene index index on region /region with analyzer StandardAnalyzer on field field1 because another member defines the same index with analyzer KeywordAnalyzer on that field."; + public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_2 = "Cannot create Lucene index index on region /region with analyzer KeywordAnalyzer on field field1 because another member defines the same index with analyzer StandardAnalyzer on that field."; + public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_ANALYZERS_3 = "Cannot create Lucene index index on region /region with analyzer KeywordAnalyzer on field field2 because another member defines the same index with analyzer StandardAnalyzer on that field."; public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_NAMES = "Cannot create Region /region with [index2#_region] async event ids because another cache has the same region defined with [index1#_region] async event ids"; public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_INDEXES_1 = "Cannot create Region /region with [] async event ids because another cache has the same region defined with [index#_region] async event ids"; public static final String CANNOT_CREATE_LUCENE_INDEX_DIFFERENT_INDEXES_2 = "Cannot create Region /region with [index#_region, index2#_region] async event ids because another cache has the same region defined with [index#_region] async event ids";
