http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexCommands.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexCommands.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexCommands.java new file mode 100755 index 0000000..200a14f --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexCommands.java @@ -0,0 +1,470 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal.cli; + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.geode.security.ResourcePermission.Operation; +import org.apache.geode.security.ResourcePermission.Resource; +import org.springframework.shell.core.annotation.CliAvailabilityIndicator; +import org.springframework.shell.core.annotation.CliCommand; +import org.springframework.shell.core.annotation.CliOption; + +import com.gemstone.gemfire.SystemFailure; +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.execute.Execution; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionInvocationTargetException; +import com.gemstone.gemfire.cache.execute.ResultCollector; + +import com.gemstone.gemfire.cache.lucene.internal.cli.functions.LuceneCreateIndexFunction; +import com.gemstone.gemfire.cache.lucene.internal.cli.functions.LuceneDescribeIndexFunction; +import com.gemstone.gemfire.cache.lucene.internal.cli.functions.LuceneListIndexFunction; +import com.gemstone.gemfire.cache.lucene.internal.cli.functions.LuceneSearchIndexFunction; +import com.gemstone.gemfire.distributed.DistributedMember; +import com.gemstone.gemfire.internal.cache.execute.AbstractExecution; +import com.gemstone.gemfire.internal.security.IntegratedSecurityService; +import com.gemstone.gemfire.internal.security.SecurityService; +import com.gemstone.gemfire.management.cli.CliMetaData; +import com.gemstone.gemfire.management.cli.ConverterHint; +import com.gemstone.gemfire.management.cli.Result; +import com.gemstone.gemfire.management.internal.cli.CliUtil; +import com.gemstone.gemfire.management.internal.cli.commands.AbstractCommandsSupport; +import com.gemstone.gemfire.management.internal.cli.functions.CliFunctionResult; +import com.gemstone.gemfire.management.internal.cli.i18n.CliStrings; +import com.gemstone.gemfire.management.internal.cli.result.CommandResult; +import com.gemstone.gemfire.management.internal.cli.result.CommandResultException; +import com.gemstone.gemfire.management.internal.cli.result.ResultBuilder; +import com.gemstone.gemfire.management.internal.cli.result.TabularResultData; +import com.gemstone.gemfire.management.internal.cli.shell.Gfsh; +import com.gemstone.gemfire.management.internal.configuration.domain.XmlEntity; +import com.gemstone.gemfire.management.internal.security.ResourceOperation; + +/** + * The LuceneIndexCommands class encapsulates all Geode shell (Gfsh) commands related to Lucene indexes defined in Geode. + * </p> + * @see AbstractCommandsSupport + * @see LuceneIndexDetails + * @see LuceneListIndexFunction + */ +@SuppressWarnings("unused") +public class LuceneIndexCommands extends AbstractCommandsSupport { + private static final LuceneCreateIndexFunction createIndexFunction = new LuceneCreateIndexFunction(); + private static final LuceneDescribeIndexFunction describeIndexFunction = new LuceneDescribeIndexFunction(); + private static final LuceneSearchIndexFunction searchIndexFunction = new LuceneSearchIndexFunction(); + private List<LuceneSearchResults> searchResults=null; + + private SecurityService securityService = IntegratedSecurityService.getSecurityService(); + + @CliCommand(value = LuceneCliStrings.LUCENE_LIST_INDEX, help = LuceneCliStrings.LUCENE_LIST_INDEX__HELP) + @CliMetaData(shellOnly = false, relatedTopic={CliStrings.TOPIC_GEODE_REGION, CliStrings.TOPIC_GEODE_DATA }) + @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ) + public Result listIndex( + @CliOption(key = LuceneCliStrings.LUCENE_LIST_INDEX__STATS, + mandatory=false, + specifiedDefaultValue = "true", + unspecifiedDefaultValue = "false", + help = LuceneCliStrings.LUCENE_LIST_INDEX__STATS__HELP) final boolean stats) { + + try { + return toTabularResult(getIndexListing(),stats); + } + catch (FunctionInvocationTargetException ignore) { + return ResultBuilder.createGemFireErrorResult(CliStrings.format(CliStrings.COULD_NOT_EXECUTE_COMMAND_TRY_AGAIN, + LuceneCliStrings.LUCENE_LIST_INDEX)); + } + catch (VirtualMachineError e) { + SystemFailure.initiateFailure(e); + throw e; + } + catch (Throwable t) { + SystemFailure.checkFailure(); + getCache().getLogger().info(t); + return ResultBuilder.createGemFireErrorResult(String.format(LuceneCliStrings.LUCENE_LIST_INDEX__ERROR_MESSAGE, + toString(t, isDebugging()))); + } + } + + @SuppressWarnings("unchecked") + protected List<LuceneIndexDetails> getIndexListing() { + final Execution functionExecutor = getMembersFunctionExecutor(getMembers(getCache())); + + if (functionExecutor instanceof AbstractExecution) { + ((AbstractExecution) functionExecutor).setIgnoreDepartedMembers(true); + } + + final ResultCollector resultsCollector = functionExecutor.execute(new LuceneListIndexFunction()); + final List<Set<LuceneIndexDetails>> results = (List<Set<LuceneIndexDetails>>) resultsCollector.getResult(); + + return results.stream() + .flatMap(set -> set.stream()) + .sorted() + .collect(Collectors.toList()); + } + + protected Result toTabularResult(final List<LuceneIndexDetails> indexDetailsList, boolean stats) { + if (!indexDetailsList.isEmpty()) { + final TabularResultData indexData = ResultBuilder.createTabularResultData(); + + for (final LuceneIndexDetails indexDetails : indexDetailsList) { + indexData.accumulate("Index Name", indexDetails.getIndexName()); + indexData.accumulate("Region Path", indexDetails.getRegionPath()); + indexData.accumulate("Indexed Fields", indexDetails.getSearchableFieldNamesString()); + indexData.accumulate("Field Analyzer", indexDetails.getFieldAnalyzersString()); + indexData.accumulate("Status", indexDetails.getInitialized() == true ? "Initialized" : "Defined"); + + if (stats == true) { + if (!indexDetails.getInitialized()) { + indexData.accumulate("Query Executions", "NA"); + indexData.accumulate("Updates", "NA"); + indexData.accumulate("Commits", "NA"); + indexData.accumulate("Documents", "NA"); + } + else { + indexData.accumulate("Query Executions", indexDetails.getIndexStats().get("queryExecutions")); + indexData.accumulate("Updates", indexDetails.getIndexStats().get("updates")); + indexData.accumulate("Commits", indexDetails.getIndexStats().get("commits")); + indexData.accumulate("Documents", indexDetails.getIndexStats().get("documents")); + } + } + } + return ResultBuilder.buildResult(indexData); + } + else { + return ResultBuilder.createInfoResult(LuceneCliStrings.LUCENE_LIST_INDEX__INDEXES_NOT_FOUND_MESSAGE); + } + } + + @CliCommand(value = LuceneCliStrings.LUCENE_CREATE_INDEX, help = LuceneCliStrings.LUCENE_CREATE_INDEX__HELP) + @CliMetaData(shellOnly = false, relatedTopic={CliStrings.TOPIC_GEODE_REGION, CliStrings.TOPIC_GEODE_DATA }, writesToSharedConfiguration=true) + //TODO : Add optionContext for indexName + public Result createIndex( + @CliOption(key = LuceneCliStrings.LUCENE__INDEX_NAME, + mandatory=true, + help = LuceneCliStrings.LUCENE_CREATE_INDEX__NAME__HELP) final String indexName, + + @CliOption (key = LuceneCliStrings.LUCENE__REGION_PATH, + mandatory = true, + optionContext = ConverterHint.REGIONPATH, + help = LuceneCliStrings.LUCENE_CREATE_INDEX__REGION_HELP) final String regionPath, + + @CliOption(key = LuceneCliStrings.LUCENE_CREATE_INDEX__FIELD, + mandatory = true, + help = LuceneCliStrings.LUCENE_CREATE_INDEX__FIELD_HELP) + @CliMetaData (valueSeparator = ",") final String[] fields, + + @CliOption(key = LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER, + mandatory = false, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + help = LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER_HELP) + @CliMetaData (valueSeparator = ",") final String[] analyzers, + + @CliOption (key = LuceneCliStrings.LUCENE_CREATE_INDEX__GROUP, + optionContext = ConverterHint.MEMBERGROUP, + unspecifiedDefaultValue = CliMetaData.ANNOTATION_NULL_VALUE, + help = LuceneCliStrings.LUCENE_CREATE_INDEX__GROUP__HELP) + @CliMetaData (valueSeparator = ",") final String[] groups) { + + Result result = null; + XmlEntity xmlEntity = null; + + this.securityService.authorizeRegionManage(regionPath); + try { + final Cache cache = getCache(); + LuceneIndexInfo indexInfo = new LuceneIndexInfo(indexName, regionPath, fields, analyzers); + final ResultCollector<?, ?> rc = this.executeFunctionOnGroups(createIndexFunction, groups, indexInfo); + final List<CliFunctionResult> funcResults = (List<CliFunctionResult>) rc.getResult(); + + final TabularResultData tabularResult = ResultBuilder.createTabularResultData(); + for (final CliFunctionResult cliFunctionResult : funcResults) { + tabularResult.accumulate("Member",cliFunctionResult.getMemberIdOrName()); + + if (cliFunctionResult.isSuccessful()) { + tabularResult.accumulate("Status","Successfully created lucene index"); +// if (xmlEntity == null) { +// xmlEntity = cliFunctionResult.getXmlEntity(); +// } + } + else { + tabularResult.accumulate("Status","Failed: "+cliFunctionResult.getMessage()); + } + } + result = ResultBuilder.buildResult(tabularResult); + } + catch (CommandResultException crex) { + result = crex.getResult(); + } catch (Exception e) { + result = ResultBuilder.createGemFireErrorResult(e.getMessage()); + } +// TODO - store in cluster config +// if (xmlEntity != null) { +// result.setCommandPersisted((new SharedConfigurationWriter()).addXmlEntity(xmlEntity, groups)); +// } + + return result; + } + + @CliCommand(value = LuceneCliStrings.LUCENE_DESCRIBE_INDEX, help = LuceneCliStrings.LUCENE_DESCRIBE_INDEX__HELP) + @CliMetaData(shellOnly = false, relatedTopic={CliStrings.TOPIC_GEODE_REGION, CliStrings.TOPIC_GEODE_DATA }) + @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ) + public Result describeIndex( + @CliOption(key = LuceneCliStrings.LUCENE__INDEX_NAME, + mandatory=true, + help = LuceneCliStrings.LUCENE_DESCRIBE_INDEX__NAME__HELP) final String indexName, + + @CliOption (key = LuceneCliStrings.LUCENE__REGION_PATH, + mandatory = true, + optionContext = ConverterHint.REGIONPATH, + help = LuceneCliStrings.LUCENE_DESCRIBE_INDEX__REGION_HELP) final String regionPath) { + try { + LuceneIndexInfo indexInfo = new LuceneIndexInfo(indexName, regionPath); + return toTabularResult(getIndexDetails(indexInfo),true); + } + catch (FunctionInvocationTargetException ignore) { + return ResultBuilder.createGemFireErrorResult(CliStrings.format(CliStrings.COULD_NOT_EXECUTE_COMMAND_TRY_AGAIN, + LuceneCliStrings.LUCENE_DESCRIBE_INDEX)); + } + catch (VirtualMachineError e) { + SystemFailure.initiateFailure(e); + throw e; + } + catch (IllegalArgumentException e) { + return ResultBuilder.createInfoResult(e.getMessage()); + } + catch (Throwable t) { + SystemFailure.checkFailure(); + getCache().getLogger().info(t); + return ResultBuilder.createGemFireErrorResult(String.format(LuceneCliStrings.LUCENE_DESCRIBE_INDEX__ERROR_MESSAGE, + toString(t, isDebugging()))); + } + } + + @SuppressWarnings("unchecked") + protected List<LuceneIndexDetails> getIndexDetails(LuceneIndexInfo indexInfo) throws Exception { + this.securityService.authorizeRegionManage(indexInfo.getRegionPath()); + final ResultCollector<?, ?> rc = this.executeFunctionOnGroups(describeIndexFunction, new String[] {}, indexInfo); + final List<LuceneIndexDetails> funcResults = (List<LuceneIndexDetails>) rc.getResult(); + return funcResults.stream().filter(indexDetails -> indexDetails != null).collect(Collectors.toList()); + } + + @CliCommand(value = LuceneCliStrings.LUCENE_SEARCH_INDEX, help = LuceneCliStrings.LUCENE_SEARCH_INDEX__HELP) + @CliMetaData(shellOnly = false, relatedTopic = { CliStrings.TOPIC_GEODE_REGION, CliStrings.TOPIC_GEODE_DATA }) + @ResourceOperation(resource = Resource.CLUSTER, operation = Operation.READ) + public Result searchIndex( + @CliOption(key = LuceneCliStrings.LUCENE__INDEX_NAME, + mandatory = true, + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__NAME__HELP) final String indexName, + + @CliOption(key = LuceneCliStrings.LUCENE__REGION_PATH, + mandatory = true, + optionContext = ConverterHint.REGIONPATH, + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__REGION_HELP) final String regionPath, + + @CliOption(key = LuceneCliStrings.LUCENE_SEARCH_INDEX__QUERY_STRING, + mandatory = true, + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__QUERY_STRING__HELP) final String queryString, + + @CliOption(key = LuceneCliStrings.LUCENE_SEARCH_INDEX__DEFAULT_FIELD, + mandatory = true, + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__DEFAULT_FIELD__HELP) final String defaultField, + + @CliOption(key = LuceneCliStrings.LUCENE_SEARCH_INDEX__LIMIT, + mandatory = false, + unspecifiedDefaultValue = "-1", + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__LIMIT__HELP) final int limit, + + @CliOption(key = LuceneCliStrings.LUCENE_SEARCH_INDEX__PAGE_SIZE, + mandatory = false, + unspecifiedDefaultValue = "-1", + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__PAGE_SIZE__HELP) int pageSize, + + @CliOption(key = LuceneCliStrings.LUCENE_SEARCH_INDEX__KEYSONLY, + mandatory = false, + unspecifiedDefaultValue = "false", + help = LuceneCliStrings.LUCENE_SEARCH_INDEX__KEYSONLY__HELP) boolean keysOnly) + { + try { + LuceneQueryInfo queryInfo = new LuceneQueryInfo(indexName, regionPath, queryString, defaultField, limit, keysOnly); + if (pageSize == -1) { + pageSize = Integer.MAX_VALUE; + } + searchResults = getSearchResults(queryInfo); + return displayResults(pageSize, keysOnly); + } + catch (FunctionInvocationTargetException ignore) { + return ResultBuilder.createGemFireErrorResult(CliStrings.format(CliStrings.COULD_NOT_EXECUTE_COMMAND_TRY_AGAIN, + LuceneCliStrings.LUCENE_SEARCH_INDEX)); + } + catch (VirtualMachineError e) { + SystemFailure.initiateFailure(e); + throw e; + } + catch (IllegalArgumentException e) { + return ResultBuilder.createInfoResult(e.getMessage()); + } + catch (Throwable t) { + SystemFailure.checkFailure(); + getCache().getLogger().info(t); + return ResultBuilder.createGemFireErrorResult(String.format(LuceneCliStrings.LUCENE_SEARCH_INDEX__ERROR_MESSAGE, + toString(t, isDebugging()))); + } + } + + private Result displayResults(int pageSize, boolean keysOnly) throws Exception { + if (searchResults.size() == 0) { + return ResultBuilder.createInfoResult(LuceneCliStrings.LUCENE_SEARCH_INDEX__NO_RESULTS_MESSAGE); + } + + Gfsh gfsh = initGfsh(); + boolean pagination = searchResults.size() > pageSize; + int fromIndex = 0; + int toIndex = pageSize < searchResults.size() ? pageSize : searchResults.size(); + int currentPage = 1; + int totalPages = (int) Math.ceil((float) searchResults.size() / pageSize); + boolean skipDisplay = false; + String step = null; + do { + + if (!skipDisplay) { + CommandResult commandResult = (CommandResult) getResults(fromIndex, toIndex, keysOnly); + if (!pagination) { + return commandResult; + } + Gfsh.println(); + while (commandResult.hasNextLine()) { + gfsh.printAsInfo(commandResult.nextLine()); + } + gfsh.printAsInfo("\t\tPage " + currentPage + " of " + totalPages); + String message = ("Press n to move to next page, q to quit and p to previous page : "); + step = gfsh.interact(message); + } + + switch (step) { + case "n": + { + if (currentPage == totalPages) { + gfsh.printAsInfo("No more results to display."); + step = gfsh.interact("Press p to move to last page and q to quit."); + skipDisplay = true; + continue; + } + + if(skipDisplay) { + skipDisplay=false; + } + else { + currentPage++; + int current = fromIndex; + fromIndex = toIndex; + toIndex = (pageSize + fromIndex >= searchResults.size()) ? searchResults.size() : pageSize + fromIndex; + } + break; + } + case "p": { + if (currentPage == 1) { + gfsh.printAsInfo("At the top of the search results."); + step = gfsh.interact("Press n to move to the first page and q to quit."); + skipDisplay=true; + continue; + } + + if (skipDisplay) { + skipDisplay = false; + } + else { + currentPage--; + int current = fromIndex; + toIndex = fromIndex; + fromIndex = current - pageSize <= 0 ? 0 : current - pageSize; + } + break; + } + case "q": + return ResultBuilder.createInfoResult("Search complete."); + default: + Gfsh.println("Invalid option"); + break; + } + } while(true); + } + + protected Gfsh initGfsh() { + return Gfsh.getCurrentInstance(); + } + + private List<LuceneSearchResults> getSearchResults(final LuceneQueryInfo queryInfo) throws Exception { + securityService.authorizeRegionManage(queryInfo.getRegionPath()); + + final String[] groups = {}; + final ResultCollector<?, ?> rc = this.executeSearch(queryInfo); + final List<Set<LuceneSearchResults>> functionResults = (List<Set<LuceneSearchResults>>) rc.getResult(); + + return functionResults.stream() + .flatMap(set -> set.stream()) + .sorted() + .collect(Collectors.toList()); + } + + private Result getResults(int fromIndex, int toIndex, boolean keysonly) throws Exception { + final TabularResultData data = ResultBuilder.createTabularResultData(); + for (int i = fromIndex; i < toIndex; i++) { + if (!searchResults.get(i).getExeptionFlag()) { + data.accumulate("key", searchResults.get(i).getKey()); + if (!keysonly) { + data.accumulate("value", searchResults.get(i).getValue()); + data.accumulate("score", searchResults.get(i).getScore()); + } + } + else { + throw new Exception(searchResults.get(i).getExceptionMessage()); + } + } + return ResultBuilder.buildResult(data); + } + + protected ResultCollector<?, ?> executeFunctionOnGroups(FunctionAdapter function, + String[] groups, + final LuceneIndexInfo indexInfo) throws IllegalArgumentException, CommandResultException + { + final Set<DistributedMember> targetMembers; + if (function != createIndexFunction) { + targetMembers = CliUtil.getMembersForeRegionViaFunction(getCache(), indexInfo.getRegionPath()); + if (targetMembers.isEmpty()) { + throw new IllegalArgumentException("Region not found."); + } + } + else { + targetMembers = CliUtil.findAllMatchingMembers(groups, null); + } + return CliUtil.executeFunction(function, indexInfo, targetMembers); + } + + protected ResultCollector<?, ?> executeSearch(final LuceneQueryInfo queryInfo) throws Exception { + final Set<DistributedMember> targetMembers = CliUtil.getMembersForeRegionViaFunction(getCache(),queryInfo.getRegionPath()); + if (targetMembers.isEmpty()) + throw new IllegalArgumentException("Region not found."); + return CliUtil.executeFunction(searchIndexFunction, queryInfo, targetMembers); + } + + @CliAvailabilityIndicator({LuceneCliStrings.LUCENE_SEARCH_INDEX, LuceneCliStrings.LUCENE_CREATE_INDEX, + LuceneCliStrings.LUCENE_DESCRIBE_INDEX, LuceneCliStrings.LUCENE_LIST_INDEX}) + public boolean indexCommandsAvailable() { + return (!CliUtil.isGfshVM() || (getGfsh() != null && getGfsh().isConnectedAndReady())); + } +}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexDetails.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexDetails.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexDetails.java new file mode 100644 index 0000000..eaec4a0 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexDetails.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal.cli; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexCreationProfile; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexStats; + +import org.apache.lucene.analysis.Analyzer; + +public class LuceneIndexDetails implements Comparable<LuceneIndexDetails>, Serializable { + private static final long serialVersionUID = 1L; + private final String indexName; + private final String regionPath; + private final String[] searchableFieldNames; + private Map<String, String> fieldAnalyzers=null; + private final Map<String,Integer> indexStats; + private boolean initialized; + + public LuceneIndexDetails(final String indexName, final String regionPath, final String[] searchableFieldNames, final Map<String, Analyzer> fieldAnalyzers, LuceneIndexStats indexStats, boolean initialized) { + this.indexName = indexName; + this.regionPath = regionPath; + this.searchableFieldNames = searchableFieldNames; + this.fieldAnalyzers = getFieldAnalyzerStrings(fieldAnalyzers); + this.indexStats=getIndexStatsMap(indexStats); + this.initialized = initialized; + } + + public LuceneIndexDetails(LuceneIndexImpl index) { + this(index.getName(), index.getRegionPath(), index.getFieldNames(),index.getFieldAnalyzers(),index.getIndexStats(), true); + } + + public LuceneIndexDetails(LuceneIndexCreationProfile indexProfile) { + this(indexProfile.getIndexName(), indexProfile.getRegionPath(), indexProfile.getFieldNames(), null, null, false); + this.fieldAnalyzers=getFieldAnalyzerStringsFromProfile(indexProfile.getFieldAnalyzers()); + } + + public Map<String,Integer> getIndexStats() { + return indexStats; + } + private Map<String,Integer> getIndexStatsMap(LuceneIndexStats indexStats) { + Map<String,Integer> statsMap = new HashMap<>(); + if (indexStats==null) return statsMap; + statsMap.put("queryExecutions",indexStats.getQueryExecutions()); + statsMap.put("updates",indexStats.getUpdates()); + statsMap.put("commits",indexStats.getCommits()); + statsMap.put("documents",indexStats.getDocuments()); + return statsMap; + } + + public String getIndexStatsString() { + return indexStats.toString(); + } + + private Map<String, String> getFieldAnalyzerStrings(Map<String, Analyzer> fieldAnalyzers) { + if(fieldAnalyzers == null) { + return Collections.emptyMap(); + } + + Map<String, String> results = new HashMap<>(); + + for (Entry<String, Analyzer> entry : fieldAnalyzers.entrySet()) { + final Analyzer analyzer = entry.getValue(); + if(analyzer != null) { + results.put(entry.getKey(), analyzer.getClass().getSimpleName()); + } + } + return results; + } + + private Map<String, String> getFieldAnalyzerStringsFromProfile(Map<String, String> fieldAnalyzers) { + if(fieldAnalyzers == null) { + return Collections.emptyMap(); + + } + + Map<String, String> results = new HashMap<>(); + + for (Entry<String, String> entry : fieldAnalyzers.entrySet()) { + final String analyzer = entry.getValue(); + if(analyzer != null) { + results.put(entry.getKey(), analyzer); + } + } + return results; + } + + public String getSearchableFieldNamesString() { + return Arrays.asList(searchableFieldNames).toString(); + } + + + public String getFieldAnalyzersString() { + return fieldAnalyzers.toString(); + } + + @Override + public String toString() { + final StringBuffer buffer = new StringBuffer(); + buffer.append("{\n\tIndex Name = "+indexName); + buffer.append(",\tRegion Path = "+regionPath); + buffer.append(",\tIndexed Fields = "+getSearchableFieldNamesString()); + buffer.append(",\tField Analyzer = "+getFieldAnalyzersString()); + buffer.append(",\tStatus =\n\t"+ getInitialized()); + buffer.append(",\tIndex Statistics =\n\t"+getIndexStatsString()); + buffer.append("\n}\n"); + return buffer.toString(); + } + + public boolean getInitialized() { + return initialized; + } + + public String getIndexName() { + return indexName; + } + + public String getRegionPath() { + return regionPath; + } + + private static <T extends Comparable<T>> int compare(final T obj1, final T obj2) { + return (obj1 == null && obj2 == null ? 0 : (obj1 == null ? 1 : (obj2 == null ? -1 : obj1.compareTo(obj2)))); + } + + @Override + public int compareTo(final LuceneIndexDetails indexDetails) { + int comparisonValue = compare(getIndexName(), indexDetails.getIndexName()); + return (comparisonValue != 0 ? comparisonValue : compare(getRegionPath(), indexDetails.getRegionPath())); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexInfo.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexInfo.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexInfo.java new file mode 100644 index 0000000..3df292c --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneIndexInfo.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gemstone.gemfire.cache.lucene.internal.cli; + +import java.io.Serializable; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; + +import org.apache.lucene.analysis.Analyzer; + +public class LuceneIndexInfo implements Serializable { + private static final long serialVersionUID = 1L; + + private final String indexName; + private final String regionPath; + private final String[] searchableFieldNames; + private final String[] fieldAnalyzers; + + public LuceneIndexInfo(final String indexName, final String regionPath, final String[] searchableFieldNames, String[] fieldAnalyzers) { + this.indexName = indexName; + this.regionPath = regionPath; + this.searchableFieldNames = searchableFieldNames; + this.fieldAnalyzers = fieldAnalyzers; + } + + public LuceneIndexInfo(final String indexName, final String regionPath) { + this(indexName,regionPath,null,null); + } + + public String getIndexName() { + return indexName; + } + + public String getRegionPath() { + return regionPath; + } + + public String[] getSearchableFieldNames() { + return searchableFieldNames; + } + + public String[] getFieldAnalyzers() { + return fieldAnalyzers; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneQueryInfo.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneQueryInfo.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneQueryInfo.java new file mode 100644 index 0000000..0dbe2fe --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneQueryInfo.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli; + +import java.io.Serializable; + +import com.gemstone.gemfire.cache.lucene.LuceneQueryFactory; + +public class LuceneQueryInfo implements Serializable { + private static final long serialVersionUID = 1L; + private String indexName; + private String regionPath; + private String queryString; + private String defaultField; + private int limit; + private boolean keysOnly; + + public LuceneQueryInfo(final String indexName, + final String regionPath, + final String queryString, + final String defaultField, + final int limit, + final boolean keysOnly) + { + this.indexName = indexName; + this.regionPath = regionPath; + this.queryString = queryString; + this.defaultField = defaultField; + this.limit = limit; + this.keysOnly = keysOnly; + } + + public String getIndexName() { + return indexName; + } + + public String getRegionPath() { + return regionPath; + } + + public String getQueryString() { + return queryString; + } + + public String getDefaultField() { + return defaultField; + } + + public int getLimit() { + if (limit == -1) return LuceneQueryFactory.DEFAULT_LIMIT; + else return limit; } + + public boolean getKeysOnly() { return keysOnly; } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneSearchResults.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneSearchResults.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneSearchResults.java new file mode 100644 index 0000000..9f03873 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/LuceneSearchResults.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli; + +import java.io.Serializable; + +public class LuceneSearchResults<K,V> implements Comparable<LuceneSearchResults>, Serializable { + + private String key; + private String value; + private float score; + private boolean exceptionFlag = false; + private String exceptionMessage; + + + public LuceneSearchResults(final String key, final String value, final float score) { + this.key = key; + this.value = value; + this.score = score; + } + + public LuceneSearchResults(final String key) { + this.key = key; + } + + public LuceneSearchResults(final boolean exceptionFlag, final String exceptionMessage) { + this.exceptionFlag=exceptionFlag; + this.exceptionMessage=exceptionMessage; + } + + public String getKey() { + return key; + } + + public String getValue() { + return value; + } + + public float getScore() { + return score; + } + + @Override + public int compareTo(final LuceneSearchResults searchResults) { + return Float.compare(getScore(),searchResults.getScore()); + } + + public boolean getExeptionFlag() { return exceptionFlag; } + + public String getExceptionMessage() { return exceptionMessage; } + + @Override public String toString() { + return "LuceneSearchResults{" + + "key='" + key + '\'' + + ", value='" + value + '\'' + + ", score=" + score + + ", exceptionFlag=" + exceptionFlag + + ", exceptionMessage='" + exceptionMessage + '\'' + + '}'; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java new file mode 100644 index 0000000..cb45f7b --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneCreateIndexFunction.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli.functions; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.CacheFactory; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneCliStrings; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexDetails; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexInfo; +import com.gemstone.gemfire.internal.InternalEntity; +import com.gemstone.gemfire.internal.lang.StringUtils; +import com.gemstone.gemfire.management.internal.cli.CliUtil; +import com.gemstone.gemfire.management.internal.cli.functions.CliFunctionResult; +import com.gemstone.gemfire.management.internal.cli.i18n.CliStrings; +import com.gemstone.gemfire.management.internal.configuration.domain.XmlEntity; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; + + +/** + * The LuceneCreateIndexFunction class is a function used to create Lucene indexes. + * </p> + * @see Cache + * @see com.gemstone.gemfire.cache.execute.Function + * @see FunctionAdapter + * @see FunctionContext + * @see InternalEntity + * @see LuceneIndexDetails + */ +@SuppressWarnings("unused") +public class LuceneCreateIndexFunction extends FunctionAdapter implements InternalEntity { + + protected Cache getCache() { + return CacheFactory.getAnyInstance(); + } + + public String getId() { + return LuceneListIndexFunction.class.getName(); + } + + public void execute(final FunctionContext context) { + String memberId = null; + try { + final LuceneIndexInfo indexInfo = (LuceneIndexInfo) context.getArguments(); + final Cache cache = getCache(); + memberId = cache.getDistributedSystem().getDistributedMember().getId(); + LuceneService service = LuceneServiceProvider.get(cache); + + String[] fields = indexInfo.getSearchableFieldNames(); + String[] analyzerName = indexInfo.getFieldAnalyzers(); + + if (analyzerName == null || analyzerName.length == 0) { + service.createIndex(indexInfo.getIndexName(), indexInfo.getRegionPath(), fields); + } + else { + if (analyzerName.length != fields.length) throw new Exception("Mismatch in lengths of fields and analyzers"); + Map<String, Analyzer> fieldAnalyzer = new HashMap<>(); + for (int i = 0; i < fields.length; i++) { + Analyzer analyzer = toAnalyzer(analyzerName[i]); + fieldAnalyzer.put(fields[i], analyzer); + } + service.createIndex(indexInfo.getIndexName(), indexInfo.getRegionPath(), fieldAnalyzer); + } + + //TODO - update cluster configuration by returning a valid XmlEntity + XmlEntity xmlEntity = null; + context.getResultSender().lastResult(new CliFunctionResult(memberId, xmlEntity)); + } + catch (Exception e) { + String exceptionMessage = CliStrings.format(CliStrings.EXCEPTION_CLASS_AND_MESSAGE, e.getClass().getName(), + e.getMessage()); + context.getResultSender().lastResult(new CliFunctionResult(memberId, e, e.getMessage())); + } + } + + private Analyzer toAnalyzer(String className) + { + if (className==null) + className=StandardAnalyzer.class.getCanonicalName(); + else if (StringUtils.trim(className).equals("") | StringUtils.trim(className).equals("null") ) + className = StandardAnalyzer.class.getCanonicalName(); + + Class<? extends Analyzer> clazz = CliUtil.forName(className, LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER); + return CliUtil.newInstance(clazz, LuceneCliStrings.LUCENE_CREATE_INDEX__ANALYZER); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneDescribeIndexFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneDescribeIndexFunction.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneDescribeIndexFunction.java new file mode 100755 index 0000000..015828a --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneDescribeIndexFunction.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli.functions; + +import java.util.HashSet; +import java.util.Set; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.CacheFactory; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.lucene.LuceneIndex; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexCreationProfile; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; +import com.gemstone.gemfire.cache.lucene.internal.LuceneServiceImpl; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexDetails; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexInfo; +import com.gemstone.gemfire.internal.InternalEntity; + +/** + * The LuceneDescribeIndexFunction class is a function used to collect the information on a particular lucene index. + * </p> + * @see Cache + * @see com.gemstone.gemfire.cache.execute.Function + * @see FunctionAdapter + * @see FunctionContext + * @see InternalEntity + * @see LuceneIndexDetails + * @see LuceneIndexInfo + */ +@SuppressWarnings("unused") +public class LuceneDescribeIndexFunction extends FunctionAdapter implements InternalEntity { + + protected Cache getCache() { + return CacheFactory.getAnyInstance(); + } + + public String getId() { + return LuceneDescribeIndexFunction.class.getName(); + } + + public void execute(final FunctionContext context) { + LuceneIndexDetails result = null; + + final Cache cache = getCache(); + final LuceneIndexInfo indexInfo = (LuceneIndexInfo) context.getArguments(); + LuceneServiceImpl service = (LuceneServiceImpl) LuceneServiceProvider.get(cache); + LuceneIndex index = service.getIndex(indexInfo.getIndexName(), indexInfo.getRegionPath()); + LuceneIndexCreationProfile profile = service.getDefinedIndex(indexInfo.getIndexName(),indexInfo.getRegionPath()); + if (index != null) { + result = new LuceneIndexDetails((LuceneIndexImpl) index); + } else if (profile != null) { + result = new LuceneIndexDetails(profile); + } + context.getResultSender().lastResult(result); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneListIndexFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneListIndexFunction.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneListIndexFunction.java new file mode 100755 index 0000000..bb74410 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneListIndexFunction.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli.functions; + +import java.util.HashSet; +import java.util.Set; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.CacheFactory; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.lucene.LuceneIndex; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexCreationProfile; +import com.gemstone.gemfire.cache.lucene.internal.LuceneIndexImpl; +import com.gemstone.gemfire.cache.lucene.internal.LuceneServiceImpl; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexDetails; +import com.gemstone.gemfire.internal.InternalEntity; + +/** + * The LuceneListIndexFunction class is a function used to collect the information on all lucene indexes in + * the entire Cache. + * </p> + * @see Cache + * @see com.gemstone.gemfire.cache.execute.Function + * @see FunctionAdapter + * @see FunctionContext + * @see InternalEntity + * @see LuceneIndexDetails + */ +@SuppressWarnings("unused") +public class LuceneListIndexFunction extends FunctionAdapter implements InternalEntity { + + protected Cache getCache() { + return CacheFactory.getAnyInstance(); + } + + public String getId() { + return LuceneListIndexFunction.class.getName(); + } + + public void execute(final FunctionContext context) { + final Set<LuceneIndexDetails> indexDetailsSet = new HashSet<>(); + final Cache cache = getCache(); + LuceneServiceImpl service = (LuceneServiceImpl) LuceneServiceProvider.get(cache); + for (LuceneIndex index : service.getAllIndexes()) { + indexDetailsSet.add(new LuceneIndexDetails((LuceneIndexImpl) index)); + } + + for(LuceneIndexCreationProfile profile : service.getAllDefinedIndexes()) { + indexDetailsSet.add(new LuceneIndexDetails(profile)); + } + context.getResultSender().lastResult(indexDetailsSet); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneSearchIndexFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneSearchIndexFunction.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneSearchIndexFunction.java new file mode 100755 index 0000000..234583c --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/cli/functions/LuceneSearchIndexFunction.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.cli.functions; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import com.gemstone.gemfire.cache.Cache; +import com.gemstone.gemfire.cache.CacheFactory; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.lucene.LuceneQuery; +import com.gemstone.gemfire.cache.lucene.LuceneQueryException; +import com.gemstone.gemfire.cache.lucene.LuceneResultStruct; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.PageableLuceneQueryResults; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexDetails; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneIndexInfo; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneQueryInfo; +import com.gemstone.gemfire.cache.lucene.internal.cli.LuceneSearchResults; +import com.gemstone.gemfire.cache.query.RegionNotFoundException; +import com.gemstone.gemfire.internal.InternalEntity; + +/** + * The LuceneSearchIndexFunction class is a function used to collect the information on a particular lucene index. + * </p> + * @see Cache + * @see com.gemstone.gemfire.cache.execute.Function + * @see FunctionAdapter + * @see FunctionContext + * @see InternalEntity + * @see LuceneIndexDetails + * @see LuceneIndexInfo + */ +@SuppressWarnings("unused") +public class LuceneSearchIndexFunction<K, V> extends FunctionAdapter implements InternalEntity { + + protected Cache getCache() { + return CacheFactory.getAnyInstance(); + } + + public String getId() { + return LuceneSearchIndexFunction.class.getName(); + } + + public void execute(final FunctionContext context) { + Set<LuceneSearchResults> result = new HashSet<>(); + final Cache cache = getCache(); + final LuceneQueryInfo queryInfo = (LuceneQueryInfo) context.getArguments(); + + LuceneService luceneService = LuceneServiceProvider.get(getCache()); + try { + if (luceneService.getIndex(queryInfo.getIndexName(), queryInfo.getRegionPath()) == null) { + throw new Exception("Index " + queryInfo.getIndexName() + " not found on region " + queryInfo.getRegionPath()); + } + final LuceneQuery<K, V> query = luceneService.createLuceneQueryFactory() + .setResultLimit(queryInfo.getLimit()) + .create(queryInfo.getIndexName(), queryInfo.getRegionPath(), queryInfo.getQueryString(), + queryInfo.getDefaultField()); + if (queryInfo.getKeysOnly()) { + query.findKeys().forEach(key -> result.add(new LuceneSearchResults(key.toString()))); + } + else { + PageableLuceneQueryResults pageableLuceneQueryResults = query.findPages(); + while (pageableLuceneQueryResults.hasNext()) { + List<LuceneResultStruct> page = pageableLuceneQueryResults.next(); + page.stream() + .forEach(searchResult -> + result.add( + new LuceneSearchResults<K, V>(searchResult.getKey().toString(), searchResult.getValue().toString(), + searchResult.getScore()))); + } + } + if (result != null) { + context.getResultSender().lastResult(result); + } + } + catch (LuceneQueryException e) { + result.add(new LuceneSearchResults(true, e.getRootCause().getMessage())); + context.getResultSender().lastResult(result); + } + catch (Exception e) { + result.add(new LuceneSearchResults(true, e.getMessage())); + context.getResultSender().lastResult(result); + } + } +} + http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/DumpDirectoryFiles.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/DumpDirectoryFiles.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/DumpDirectoryFiles.java new file mode 100644 index 0000000..0d18144 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/DumpDirectoryFiles.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.directory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.execute.Function; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.execute.FunctionException; +import com.gemstone.gemfire.cache.execute.RegionFunctionContext; +import com.gemstone.gemfire.cache.execute.ResultSender; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.internal.InternalLuceneIndex; +import com.gemstone.gemfire.cache.lucene.internal.LuceneServiceImpl; +import com.gemstone.gemfire.cache.lucene.internal.distributed.CollectorManager; +import com.gemstone.gemfire.cache.lucene.internal.distributed.LuceneFunctionContext; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollector; +import com.gemstone.gemfire.cache.lucene.internal.distributed.TopEntriesCollectorManager; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystem; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepositoryImpl; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexResultCollector; +import com.gemstone.gemfire.cache.lucene.internal.repository.RepositoryManager; +import com.gemstone.gemfire.cache.query.QueryException; +import com.gemstone.gemfire.internal.InternalEntity; +import com.gemstone.gemfire.internal.cache.BucketNotFoundException; +import com.gemstone.gemfire.internal.logging.LogService; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; + +public class DumpDirectoryFiles implements Function, InternalEntity { + private static final long serialVersionUID = 1L; + + private static final Logger logger = LogService.getLogger(); + public static final String ID = DumpDirectoryFiles.class.getSimpleName(); + + @Override + public void execute(FunctionContext context) { + RegionFunctionContext ctx = (RegionFunctionContext) context; + + if(!(context.getArguments() instanceof String[])) { + throw new IllegalArgumentException("Arguments should be a string array"); + } + String[] args = (String[]) context.getArguments(); + if(args.length != 2) { + throw new IllegalArgumentException("Expected 2 arguments: exportLocation, indexName"); + } + + + String exportLocation =args[0]; + String indexName =args[1]; + + final Region<Object, Object> region = ctx.getDataSet(); + LuceneService service = LuceneServiceProvider.get(ctx.getDataSet().getCache()); + InternalLuceneIndex index = (InternalLuceneIndex) service.getIndex(indexName, region.getFullPath()); + if(index == null) { + throw new IllegalStateException("Index not found for region " + region + " index " + indexName); + } + + final RepositoryManager repoManager = index.getRepositoryManager(); + try { + final Collection<IndexRepository> repositories = repoManager.getRepositories(ctx); + repositories.stream().forEach(repo -> { + final IndexWriter writer = repo.getWriter(); + RegionDirectory directory = (RegionDirectory) writer.getDirectory(); + FileSystem fs = directory.getFileSystem(); + + String bucketName = index.getName() + "_" + repo.getRegion().getFullPath(); + bucketName = bucketName.replace("/", "_"); + File bucketDirectory = new File(exportLocation, bucketName); + bucketDirectory.mkdirs(); + fs.export(bucketDirectory); + }); + context.getResultSender().lastResult(null); + } + catch (BucketNotFoundException e) { + throw new FunctionException(e); + } + } + + @Override public String getId() { + return ID; + } + + @Override + public boolean optimizeForWrite() { + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/FileIndexInput.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/FileIndexInput.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/FileIndexInput.java new file mode 100644 index 0000000..9ebef51 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/FileIndexInput.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.directory; + +import java.io.EOFException; +import java.io.IOException; + +import org.apache.lucene.store.IndexInput; + +import com.gemstone.gemfire.cache.lucene.internal.filesystem.File; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.SeekableInputStream; + +final class FileIndexInput extends IndexInput { + private final File file; + SeekableInputStream in; + private long position; + + //Used for slice operations + private long sliceOffset; + private long sliceLength; + + FileIndexInput(String resourceDesc, File file) { + this(resourceDesc, file, 0L, file.getLength()); + } + + /** + * Constructor for a slice. + */ + private FileIndexInput(String resourceDesc, File file, long offset, long length) { + super(resourceDesc); + this.file = file; + in = file.getInputStream(); + this.sliceOffset = offset; + this.sliceLength = length; + } + + @Override + public long length() { + return sliceLength; + } + + @Override + public void close() throws IOException { + in.close(); + } + + @Override + public FileIndexInput clone() { + FileIndexInput clone = (FileIndexInput)super.clone(); + clone.in = in.clone(); + return clone; + } + + @Override + public long getFilePointer() { + return position; + } + + @Override + public void seek(long pos) throws IOException { + in.seek(pos + sliceOffset); + this.position = pos; + } + + @Override + public IndexInput slice(String sliceDescription, long offset, long length) + throws IOException { + if(length > (this.sliceLength - offset)) { + throw new IllegalArgumentException("Slice length is to large. Asked for " + length + " file length is " + sliceLength + ": " + this.file.getName()); + } + if(offset < 0 || offset >= this.sliceLength) { + throw new IllegalArgumentException("Slice offset is invalid: " + this.file.getName()); + } + + FileIndexInput result = new FileIndexInput(sliceDescription, file, sliceOffset + offset, length); + result.seek(0); + return result; + } + + @Override + public byte readByte() throws IOException { + if(++position > sliceLength) { + throw new EOFException("Read past end of file " + file.getName()); + } + + int result = in.read(); + if(result == -1) { + throw new EOFException("Read past end of file " + file.getName()); + } else { + return (byte) result; + } + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + if(len == 0) { + return; + } + + if(position + len > sliceLength) { + throw new EOFException("Read past end of file " + file.getName()); + } + + //For the FileSystemInputStream, it will always read all bytes, up + //until the end of the file. So if we didn't get enough bytes, it's + //because we reached the end of the file. + int numRead = in.read(b, offset, len); + if(numRead < len) { + throw new EOFException("Read past end of file " + file.getName()); + } + + position+=len; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/RegionDirectory.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/RegionDirectory.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/RegionDirectory.java new file mode 100644 index 0000000..45b9c97 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/RegionDirectory.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.directory; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.UUID; +import java.util.concurrent.ConcurrentMap; + +import org.apache.lucene.store.BaseDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.OutputStreamIndexOutput; +import org.apache.lucene.store.SingleInstanceLockFactory; + +import com.gemstone.gemfire.cache.lucene.internal.filesystem.ChunkKey; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.File; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystem; +import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystemStats; + +/** + * An implementation of Directory that stores data in geode regions. + * + * Directory is an interface to file/RAM storage for lucene. This class uses + * the {@link FileSystem} class to store the data in the provided geode + * regions. + */ +public class RegionDirectory extends BaseDirectory { + + private final FileSystem fs; + + /** + * Create a region directory with a given file and chunk region. These regions + * may be bucket regions or they may be replicated regions. + */ + public RegionDirectory(ConcurrentMap<String, File> fileRegion, ConcurrentMap<ChunkKey, byte[]> chunkRegion, FileSystemStats stats) { + super(new SingleInstanceLockFactory()); + fs = new FileSystem(fileRegion, chunkRegion, stats); + } + + @Override + public String[] listAll() throws IOException { + ensureOpen(); + String[] array = fs.listFileNames().toArray(new String[]{}); + Arrays.sort(array); + return array; + } + + @Override + public void deleteFile(String name) throws IOException { + ensureOpen(); + fs.deleteFile(name); + } + + @Override + public long fileLength(String name) throws IOException { + ensureOpen(); + return fs.getFile(name).getLength(); + } + + @Override + public IndexOutput createOutput(final String name, final IOContext context) throws IOException { + ensureOpen(); + final File file = fs.createFile(name); + final OutputStream out = file.getOutputStream(); + + return new OutputStreamIndexOutput(name, name, out, 1000); + } + + public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException { + String name = prefix + "_temp_" + UUID.randomUUID() + suffix; + final File file = fs.createTemporaryFile(name); + final OutputStream out = file.getOutputStream(); + + return new OutputStreamIndexOutput(name, name, out, 1000); + } + + @Override + public void sync(Collection<String> names) throws IOException { + ensureOpen(); + // Region does not need to sync to disk + } + + @Override + public void renameFile(String source, String dest) throws IOException { + ensureOpen(); + fs.renameFile(source, dest); + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + ensureOpen(); + final File file = fs.getFile(name); + + return new FileIndexInput(name, file); + } + + @Override + public void close() throws IOException { + isOpen = false; + } + + /** + * For testing, the file system + */ + public FileSystem getFileSystem() { + return fs; + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/package-info.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/package-info.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/package-info.java new file mode 100644 index 0000000..2dd0606 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/directory/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * An implementation of Lucene's {@link org.apache.lucene.store.Directory} interface that uses the filesystem + * API in {@link com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystem} + */ +package com.gemstone.gemfire.cache.lucene.internal.directory; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/CollectorManager.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/CollectorManager.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/CollectorManager.java new file mode 100644 index 0000000..4d1d1c2 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/CollectorManager.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.distributed; + +import java.io.IOException; +import java.util.Collection; + +import com.gemstone.gemfire.annotations.Experimental; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexResultCollector; + +/** + * {@link CollectorManager}s create instances of {@link IndexResultCollector} and utility methods to aggregate results + * collected by individual collectors. The collectors created by this class are primarily used for collecting results + * from {@link IndexRepository}s. The collectors can also be used for aggregating results of other collectors of same + * type. Typically search result aggregation is completed in two phases. First at a member level for merging results + * from local buckets. And then at search coordinator level for merging results from members. Use of same collector in + * both phases is expected to produce deterministic merge result irrespective of the way buckets are distributed. + * + * @param <C> Type of IndexResultCollector created by this manager + */ +@Experimental +public interface CollectorManager<C extends IndexResultCollector> { + /** + * @param name Name/Identifier for this collector. For e.g. region/bucketId. + * @return a new {@link IndexResultCollector}. This must return a different instance on + * each call. A new collector would be created for each bucket on a member node. + */ + C newCollector(String name); + + /** + * Reduce the results of individual collectors into a meaningful result. This method must be called after collection + * is finished on all provided collectors. + * + */ + C reduce(Collection<C> results); +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/EntryScore.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/EntryScore.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/EntryScore.java new file mode 100644 index 0000000..e891156 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/EntryScore.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.distributed; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import com.gemstone.gemfire.DataSerializer; +import com.gemstone.gemfire.internal.DataSerializableFixedID; +import com.gemstone.gemfire.internal.Version; + +/** + * Holds one entry matching search query and its metadata + * + * @param <K> the type of the key + */ +public class EntryScore<K> implements DataSerializableFixedID { + // Key of the entry matching search query + private K key; + + // The score of this document for the query. + private float score; + + public EntryScore() { + } + + public EntryScore(K key, float score) { + this.key = key; + this.score = score; + } + + public K getKey() { + return key; + } + + public float getScore() { + return score; + } + + @Override + public String toString() { + return "key=" + key + " score=" + score; + } + + @Override + public Version[] getSerializationVersions() { + return null; + } + + @Override + public int getDSFID() { + return LUCENE_ENTRY_SCORE; + } + + @Override + public void toData(DataOutput out) throws IOException { + DataSerializer.writeObject(key, out); + out.writeFloat(score); + } + + @Override + public void fromData(DataInput in) throws IOException, ClassNotFoundException { + key = DataSerializer.readObject(in); + score = in.readFloat(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunction.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunction.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunction.java new file mode 100644 index 0000000..3c6c0d2 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunction.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.distributed; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.Query; + +import com.gemstone.gemfire.cache.Region; +import com.gemstone.gemfire.cache.execute.FunctionAdapter; +import com.gemstone.gemfire.cache.execute.FunctionContext; +import com.gemstone.gemfire.cache.execute.FunctionException; +import com.gemstone.gemfire.cache.execute.RegionFunctionContext; +import com.gemstone.gemfire.cache.execute.ResultSender; +import com.gemstone.gemfire.cache.lucene.LuceneQueryException; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; +import com.gemstone.gemfire.cache.lucene.LuceneService; +import com.gemstone.gemfire.cache.lucene.LuceneServiceProvider; +import com.gemstone.gemfire.cache.lucene.internal.InternalLuceneIndex; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexResultCollector; +import com.gemstone.gemfire.cache.lucene.internal.repository.RepositoryManager; +import com.gemstone.gemfire.cache.query.QueryException; +import com.gemstone.gemfire.internal.InternalEntity; +import com.gemstone.gemfire.internal.cache.BucketNotFoundException; +import com.gemstone.gemfire.internal.cache.execute.BucketMovedException; +import com.gemstone.gemfire.internal.logging.LogService; + +/** + * {@link LuceneFunction} coordinates text search on a member. It receives text search query from the coordinator + * and arguments like region and buckets. It invokes search on the local index and provides a result collector. The + * locally collected results are sent to the search coordinator. + */ +public class LuceneFunction extends FunctionAdapter implements InternalEntity { + private static final long serialVersionUID = 1L; + public static final String ID = LuceneFunction.class.getName(); + + private static final Logger logger = LogService.getLogger(); + + @Override + public void execute(FunctionContext context) { + RegionFunctionContext ctx = (RegionFunctionContext) context; + ResultSender<TopEntriesCollector> resultSender = ctx.getResultSender(); + + Region region = ctx.getDataSet(); + + LuceneFunctionContext<IndexResultCollector> searchContext = (LuceneFunctionContext) ctx.getArguments(); + if (searchContext == null) { + throw new IllegalArgumentException("Missing search context"); + } + + LuceneQueryProvider queryProvider = searchContext.getQueryProvider(); + if (queryProvider == null) { + throw new IllegalArgumentException("Missing query provider"); + } + + LuceneService service = LuceneServiceProvider.get(region.getCache()); + InternalLuceneIndex index = (InternalLuceneIndex) service.getIndex(searchContext.getIndexName(), region.getFullPath()); + RepositoryManager repoManager = index.getRepositoryManager(); + + Query query = null; + try { + query = queryProvider.getQuery(index); + } catch (LuceneQueryException e) { + logger.warn("", e); + throw new FunctionException(e); + } + + if (logger.isDebugEnabled()) { + logger.debug("Executing lucene query: {}, on region {}", query, region.getFullPath()); + } + + int resultLimit = searchContext.getLimit(); + CollectorManager manager = (searchContext == null) ? null : searchContext.getCollectorManager(); + if (manager == null) { + manager = new TopEntriesCollectorManager(null, resultLimit); + } + + Collection<IndexResultCollector> results = new ArrayList<>(); + try { + Collection<IndexRepository> repositories = repoManager.getRepositories(ctx); + for (IndexRepository repo : repositories) { + IndexResultCollector collector = manager.newCollector(repo.toString()); + logger.debug("Executing search on repo: " + repo.toString()); + repo.query(query, resultLimit, collector); + results.add(collector); + } + TopEntriesCollector mergedResult = (TopEntriesCollector) manager.reduce(results); + resultSender.lastResult(mergedResult); + } catch (IOException|BucketNotFoundException e) { + logger.warn("", e); + throw new FunctionException(e); + } + } + + + @Override + public String getId() { + return ID; + } + + @Override + public boolean optimizeForWrite() { + return true; + } +} http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/05e6d966/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunctionContext.java ---------------------------------------------------------------------- diff --git a/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunctionContext.java b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunctionContext.java new file mode 100644 index 0000000..b0b2c60 --- /dev/null +++ b/geode-lucene/src/main/java/org/apache/geode/cache/lucene/internal/distributed/LuceneFunctionContext.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.gemstone.gemfire.cache.lucene.internal.distributed; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import com.gemstone.gemfire.DataSerializer; +import com.gemstone.gemfire.cache.lucene.LuceneQueryFactory; +import com.gemstone.gemfire.cache.lucene.LuceneQueryProvider; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexRepository; +import com.gemstone.gemfire.cache.lucene.internal.repository.IndexResultCollector; +import com.gemstone.gemfire.internal.DataSerializableFixedID; +import com.gemstone.gemfire.internal.Version; + +/** + * Contains function arguments for text / lucene search + */ +public class LuceneFunctionContext<C extends IndexResultCollector> implements DataSerializableFixedID { + private CollectorManager<C> manager; + private int limit; + private LuceneQueryProvider queryProvider; + private String indexName; + + public LuceneFunctionContext() { + this(null, null, null); + } + + public LuceneFunctionContext(LuceneQueryProvider provider, String indexName) { + this(provider, indexName, null); + } + + public LuceneFunctionContext(LuceneQueryProvider provider, String indexName, CollectorManager<C> manager) { + this(provider, indexName, manager, LuceneQueryFactory.DEFAULT_LIMIT); + } + + public LuceneFunctionContext(LuceneQueryProvider provider, String indexName, CollectorManager<C> manager, int limit) { + this.queryProvider = provider; + this.indexName = indexName; + this.manager = manager; + this.limit = limit; + } + + /** + * @return The maximum count of result objects to be produced by the function + */ + public int getLimit() { + return limit; + } + + /** + * Get the name of the index to query + */ + public String getIndexName() { + return indexName; + } + + /** + * On each member, search query is executed on one or more {@link IndexRepository}s. A {@link CollectorManager} could + * be provided to customize the way results from these repositories is collected and merged. + * + * @return {@link CollectorManager} instance to be used by function + */ + public CollectorManager<C> getCollectorManager() { + return this.manager; + } + + public LuceneQueryProvider getQueryProvider() { + return queryProvider; + } + + @Override + public int getDSFID() { + return LUCENE_FUNCTION_CONTEXT; + } + + @Override + public void toData(DataOutput out) throws IOException { + out.writeInt(limit); + DataSerializer.writeObject(queryProvider, out); + DataSerializer.writeObject(manager, out); + DataSerializer.writeString(indexName, out); + } + + @Override + public void fromData(DataInput in) throws IOException, ClassNotFoundException { + limit = in.readInt(); + queryProvider = DataSerializer.readObject(in); + manager = DataSerializer.readObject(in); + this.indexName = DataSerializer.readString(in); + } + + @Override + public Version[] getSerializationVersions() { + return null; + } +}
