Author: ivol37 at gmail.com Date: Mon Jan 24 16:22:35 2011 New Revision: 704
Log: initial drop-in of search and index bundle based on Solr Added: sandbox/ivol/amdatu-searchandindex/ sandbox/ivol/amdatu-searchandindex/pom.xml sandbox/ivol/amdatu-searchandindex/solr/ sandbox/ivol/amdatu-searchandindex/solr/pom.xml sandbox/ivol/amdatu-searchandindex/solr/src/ sandbox/ivol/amdatu-searchandindex/solr/src/main/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrService.java sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDispatchFilter.java sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrServiceImpl.java sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrConfig.java sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrResourceLoader.java sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/ sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/ sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/schema.xml sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/solrconfig.xml sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/solr.xml Added: sandbox/ivol/amdatu-searchandindex/pom.xml ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/pom.xml Mon Jan 24 16:22:35 2011 @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.amdatu</groupId> + <artifactId>amdatu</artifactId> + <version>0.1.0-SNAPSHOT</version> + </parent> + <artifactId>org.amdatu.searchandindex</artifactId> + <name>Amdatu Search and Index</name> + <description>This module provides search and index functionality</description> + <packaging>pom</packaging> + + <modules> + <module>solr</module> + </modules> + +</project> \ No newline at end of file Added: sandbox/ivol/amdatu-searchandindex/solr/pom.xml ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/pom.xml Mon Jan 24 16:22:35 2011 @@ -0,0 +1,118 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.amdatu</groupId> + <artifactId>org.amdatu.searchandindex</artifactId> + <version>0.1.0-SNAPSHOT</version> + </parent> + <groupId>org.amdatu.searchandindex</groupId> + <artifactId>solr</artifactId> + <packaging>bundle</packaging> + <name>Amdatu Search and Index - Solr Service</name> + <version>${platform.version}</version> + <dependencies> + <dependency> + <artifactId>solr-core</artifactId> + <groupId>org.apache.solr</groupId> + <version>1.4.1</version> + <type>jar</type> + <scope>compile</scope> + <exclusions> + <!-- Exclude woodstox lib as its groupId has bene changed from woodstox + to org.codehaus.woodstox --> + <exclusion> + <groupId>woodstox</groupId> + <artifactId>wstx-asl</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.codehaus.woodstox</groupId> + <artifactId>wstx-asl</artifactId> + <version>3.2.7</version> + <type>jar</type> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.16</version> + <scope>compile</scope> + </dependency> + + <!-- + SLF4J Binding for java.util.logging + (http://www.slf4j.org/manual.html) + --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jcl</artifactId> + <version>1.6.1</version> + </dependency> + <dependency> + <groupId>org.amdatu.web</groupId> + <artifactId>httpcontext</artifactId> + <version>${platform.version}</version> + <scope>provided</scope> + <type>bundle</type> + </dependency> + + <dependency> + <groupId>org.amdatu.authorization.login</groupId> + <artifactId>service</artifactId> + <version>${platform.version}</version> + <scope>provided</scope> + <type>bundle</type> + </dependency> + <dependency> + <groupId>org.amdatu.core</groupId> + <artifactId>tenant</artifactId> + <version>${platform.version}</version> + <scope>provided</scope> + <type>bundle</type> + </dependency> + <dependency> + <groupId>org.amdatu.core.config</groupId> + <artifactId>templates</artifactId> + <version>${platform.version}</version> + <scope>provided</scope> + <type>bundle</type> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <instructions> + <Bundle-Activator>org.amdatu.searchandindex.solr.osgi.Activator</Bundle-Activator> + <Bundle-SymbolicName>org.amdatu.searchandindex.solr</Bundle-SymbolicName> + <Embed-Dependency>*;scope=compile</Embed-Dependency> + <Embed-Transitive>true</Embed-Transitive> + <Import-Package> + !org.mortbay.*,, + !com.sun.*, + !javax.mail.*, + !javax.jmdns, + !javax.jms, + !org.relaxng.*, + !junit.framework, + !javax.portlet, + !com.ibm.*, + !org.apache.avalon.*, + !org.apache.log, + * + </Import-Package> + <Export-Package> + </Export-Package> + </instructions> + </configuration> + </plugin> + </plugins> + </build> +</project> Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrService.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/SolrService.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,39 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.searchandindex.solr; + +import org.amdatu.web.httpcontext.ResourceProvider; + +/** + * Interface for the UserService. + */ +public interface SolrService extends ResourceProvider { + /** + * The "Administrators" group. + */ + String GROUP_ADMINISTRATORS = "Administrators"; + + /** + * PID for the Solr configuration. + */ + final String PID = "org.amdatu.searchandindex.solr"; + + /** + * Configuration key for the working directory to use for Solr + */ + final String CONFIG_WORKDIR = "workdir"; +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDispatchFilter.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrDispatchFilter.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,459 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.searchandindex.solr.impl; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.NakamuraSolrConfig; +import org.apache.solr.core.NakamuraSolrResourceLoader; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.request.BinaryQueryResponseWriter; +import org.apache.solr.request.QueryResponseWriter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.servlet.SolrRequestParsers; +import org.apache.solr.servlet.cache.HttpCacheHeaderUtil; +import org.apache.solr.servlet.cache.Method; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml + * @since solr 1.2 + */ +public class SolrDispatchFilter implements Filter { + final Logger log = LoggerFactory.getLogger(SolrDispatchFilter.class); + + protected CoreContainer cores; + protected String pathPrefix; // strip this from the beginning of a path + protected String myHomeDir; // + protected String abortErrorMessage = null; + protected String solrConfigFilename = null; + protected final Map<SolrConfig, SolrRequestParsers> parsers = new WeakHashMap<SolrConfig, SolrRequestParsers>(); + // SolrRequestParsers adminRequestParser; + private final Map<String, String> myTenantMap = new HashMap<String, String>(); + + // protected final + + /** + * Constructor. + */ + public SolrDispatchFilter(String path, String homedir, Map<String, String> tenantMap) { + this.pathPrefix = path; + this.myHomeDir = homedir; + if (tenantMap != null) { + myTenantMap.putAll(tenantMap); + } + } + + /** + * {@inheritDoc} + */ + public void init(FilterConfig filterConfig) throws ServletException { + // save the current context classloader + final ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + + try { + try { + final ClassLoader classLoader = this.getClass().getClassLoader(); + + // overwrite the context classloader, restore it later + Thread.currentThread().setContextClassLoader(classLoader); + SolrResourceLoader loader = new NakamuraSolrResourceLoader(myHomeDir, classLoader); + final CoreContainer coreContainer = new CoreContainer(loader); + + if (false) { + // adminRequestParser = + // new SolrRequestParsers(new Config(null, "solr", new ByteArrayInputStream("<root/>".getBytes()), + // "")); + + String solrconfig = + "D:\\users\\dionm\\SVN\\projects\\apache-solr-1.4.1\\example\\solr\\core1\\conf\\solrconfig.xml"; + String schemaloc = + "D:\\users\\dionm\\SVN\\projects\\apache-solr-1.4.1\\example\\solr\\core1\\conf\\schema.xml"; + + File coreDir = new File("D:\\users\\dionm\\SVN\\projects\\apache-solr-1.4.1\\example\\solr\\core1"); + SolrConfig config = + new NakamuraSolrConfig(loader, "solrconfig.xml", new FileInputStream(new File(solrconfig))); + IndexSchema schema = new IndexSchema(config, null, new FileInputStream(new File(schemaloc))); + CoreDescriptor desc = new CoreDescriptor(coreContainer, "core1", coreDir.getAbsolutePath()); + SolrCore nakamuraCore = new SolrCore("core1", coreDir.getAbsolutePath(), config, schema, desc); + coreContainer.register("core1", nakamuraCore, false); + } + + this.cores = coreContainer; + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + boolean abortOnConfigurationError = true; + CoreContainer.Initializer init = createInitializer(); + try { + // web.xml configuration + init.setSolrConfigFilename(filterConfig.getInitParameter("solrconfig-filename")); + + this.cores = init.initialize(); + abortOnConfigurationError = init.isAbortOnConfigurationError(); + log.info("user.dir=" + System.getProperty("user.dir")); + } catch (Throwable t) { + // catch this so our filter still works + log.error("Could not start SOLR. Check solr/home property", t); + SolrConfig.severeErrors.add(t); + SolrCore.log(t); + } + + // Optionally abort if we found a sever error + if (abortOnConfigurationError && SolrConfig.severeErrors.size() > 0) { + StringWriter sw = new StringWriter(); + PrintWriter out = new PrintWriter(sw); + out.println("Severe errors in solr configuration.\n"); + out.println("Check your log files for more detailed information on what may be wrong.\n"); + out.println("If you want solr to continue after configuration errors, change: \n"); + out.println(" <abortOnConfigurationError>false</abortOnConfigurationError>\n"); + out.println("in " + init.getSolrConfigFilename() + "\n"); + + for (Throwable t : SolrConfig.severeErrors) { + out.println("-------------------------------------------------------------"); + t.printStackTrace(out); + } + out.flush(); + + // Servlet containers behave slightly differently if you throw an exception during + // initialization. Resin will display that error for every page, jetty prints it in + // the logs, but continues normally. (We will see a 404 rather then the real error) + // rather then leave the behavior undefined, lets cache the error and spit it out + // for every request. + abortErrorMessage = sw.toString(); + // throw new ServletException( abortErrorMessage ); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + // restore + Thread.currentThread().setContextClassLoader(contextClassLoader); + } + } + + /** Method to override to change how CoreContainer initialization is performed. */ + protected CoreContainer.Initializer createInitializer() { + return new CoreContainer.Initializer(); + } + + /** + * {@inheritDoc} + */ + public void destroy() { + if (cores != null) { + try { + cores.shutdown(); + } catch (Exception e) { + e.printStackTrace(); + } + cores = null; + } + } + + /** + * {@inheritDoc} + */ + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, + ServletException { + + if (abortErrorMessage != null) { + ((HttpServletResponse) response).sendError(500, abortErrorMessage); + return; + } + + if (request instanceof HttpServletRequest) { + HttpServletRequest req = (HttpServletRequest) request; + HttpServletResponse resp = (HttpServletResponse) response; + SolrRequestHandler handler = null; + SolrQueryRequest solrReq = null; + SolrCore core = null; + String corename = ""; + try { + // put the core container in request attribute + req.setAttribute("org.apache.solr.CoreContainer", cores); + String path = req.getServletPath(); + if (req.getPathInfo() != null) { + // this lets you handle /update/commit when /update is a servlet + path += req.getPathInfo(); + } + if (pathPrefix != null && path.startsWith(pathPrefix)) { + path = path.substring(pathPrefix.length()); + } + // check for management path + String alternate = cores.getManagementPath(); + if (alternate != null && path.startsWith(alternate)) { + path = path.substring(0, alternate.length()); + } + // unused feature ? + int idx = path.indexOf(':'); + if (idx > 0) { + // save the portion after the ':' for a 'handler' path parameter + path = path.substring(0, idx); + } + + // Check for the core admin page + if (path.equals(cores.getAdminPath())) { + handler = cores.getMultiCoreHandler(); + // solrReq = adminRequestParser.parse(null, path, req); + // handleAdminRequest(req, response, handler, solrReq); + return; + } else { + // otherwise, we should find a core from the path + idx = path.indexOf("/", 1); + if (idx > 1) { + // try to get the corename as a request parameter first + corename = path.substring(1, idx); + if ("profile".equals(corename)) { + // FIXME: hardcoded + String prefix = myTenantMap.get(request.getServerName()); + if (prefix == null) { + prefix = "Keyspace3"; + } + corename = prefix + "_" + corename; + } + core = cores.getCore(corename); + if (core != null) { + path = path.substring(idx); + } + } + if (core == null) { + corename = ""; + core = cores.getCore(""); + } + } + + // With a valid core... + if (core != null) { + final SolrConfig config = core.getSolrConfig(); + // get or create/cache the parser for the core + SolrRequestParsers parser = null; + parser = parsers.get(config); + if (parser == null) { + parser = new SolrRequestParsers(config); + parsers.put(config, parser); + } + + // Determine the handler from the url path if not set + // (we might already have selected the cores handler) + if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path + handler = core.getRequestHandler(path); + // no handler yet but allowed to handle select; let's check + if (handler == null && parser.isHandleSelect()) { + if ("/select".equals(path) || "/select/".equals(path)) { + solrReq = parser.parse(core, path, req); + String qt = solrReq.getParams().get(CommonParams.QT); + handler = core.getRequestHandler(qt); + if (handler == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + + qt); + } + } + } + } + + // With a valid handler and a valid core... + if (handler != null) { + // if not a /select, create the request + if (solrReq == null) { + solrReq = parser.parse(core, path, req); + } + + final Method reqMethod = Method.getMethod(req.getMethod()); + HttpCacheHeaderUtil.setCacheControlHeader(config, resp, reqMethod); + // unless we have been explicitly told not to, do cache validation + // if we fail cache validation, execute the query + if (config.getHttpCachingConfig().isNever304() + || !HttpCacheHeaderUtil.doCacheHeaderValidation(solrReq, req, reqMethod, resp)) { + SolrQueryResponse solrRsp = new SolrQueryResponse(); + /* + * even for HEAD requests, we need to execute the handler to ensure we don't get an error + * (and to make sure the correct QueryResponseWriter is selected and we get the correct + * Content-Type) + */ + this.execute(req, handler, solrReq, solrRsp); + HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod); + // add info to http headers + // TODO: See SOLR-232 and SOLR-267. + /* + * try { NamedList solrRspHeader = solrRsp.getResponseHeader(); for (int i=0; + * i<solrRspHeader.size(); i++) { ((javax.servlet.http.HttpServletResponse) + * response).addHeader(("Solr-" + solrRspHeader.getName(i)), + * String.valueOf(solrRspHeader.getVal(i))); } } catch (ClassCastException cce) { + * log.log(Level.WARNING, "exception adding response header log information", cce); } + */ + QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq); + writeResponse(solrRsp, response, responseWriter, solrReq, reqMethod); + } + return; // we are done with a valid handler + } + // otherwise (we have a core), let's ensure the core is in the SolrCore request attribute so + // a servlet/jsp can retrieve it + else { + req.setAttribute("org.apache.solr.SolrCore", core); + // Modify the request so each core gets its own /admin + if (path.startsWith("/admin")) { + req.getRequestDispatcher(pathPrefix == null ? path : pathPrefix + path).forward(request, + response); + return; + } + } + } + log.debug("no handler or core retrieved for " + path + ", follow through..."); + } catch (Throwable ex) { + sendError((HttpServletResponse) response, ex); + return; + } finally { + if (solrReq != null) { + solrReq.close(); + } + if (core != null) { + core.close(); + } + } + } + + // Otherwise let the webapp handle the request + chain.doFilter(request, response); + } + + private void handleAdminRequest(HttpServletRequest req, ServletResponse response, SolrRequestHandler handler, + SolrQueryRequest solrReq) throws IOException { + SolrQueryResponse solrResp = new SolrQueryResponse(); + final NamedList<Object> responseHeader = new SimpleOrderedMap<Object>(); + solrResp.add("responseHeader", responseHeader); + NamedList toLog = solrResp.getToLog(); + toLog.add("webapp", req.getContextPath()); + toLog.add("path", solrReq.getContext().get("path")); + toLog.add("params", "{" + solrReq.getParamString() + "}"); + handler.handleRequest(solrReq, solrResp); + SolrCore.setResponseHeaderValues(handler, solrReq, solrResp); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < toLog.size(); i++) { + String name = toLog.getName(i); + Object val = toLog.getVal(i); + sb.append(name).append("=").append(val).append(" "); + } + QueryResponseWriter respWriter = + SolrCore.DEFAULT_RESPONSE_WRITERS.get(solrReq.getParams().get(CommonParams.WT)); + if (respWriter == null) { + respWriter = SolrCore.DEFAULT_RESPONSE_WRITERS.get("standard"); + } + writeResponse(solrResp, response, respWriter, solrReq, Method.getMethod(req.getMethod())); + } + + private void writeResponse(SolrQueryResponse solrRsp, ServletResponse response, QueryResponseWriter responseWriter, + SolrQueryRequest solrReq, Method reqMethod) throws IOException { + if (solrRsp.getException() != null) { + sendError((HttpServletResponse) response, solrRsp.getException()); + } else { + // Now write it out + response.setContentType(responseWriter.getContentType(solrReq, solrRsp)); + if (Method.HEAD != reqMethod) { + if (responseWriter instanceof BinaryQueryResponseWriter) { + BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) responseWriter; + binWriter.write(response.getOutputStream(), solrReq, solrRsp); + } else { + PrintWriter out = response.getWriter(); + responseWriter.write(out, solrReq, solrRsp); + + } + } + // else http HEAD request, nothing to write out, waited this long just to get ContentType + } + } + + protected void execute(HttpServletRequest req, SolrRequestHandler handler, SolrQueryRequest sreq, + SolrQueryResponse rsp) { + // a custom filter could add more stuff to the request before passing it on. + // for example: sreq.getContext().put( "HttpServletRequest", req ); + // used for logging query stats in SolrCore.execute() + sreq.getContext().put("webapp", req.getContextPath()); + sreq.getCore().execute(handler, sreq, rsp); + } + + protected void sendError(HttpServletResponse res, Throwable ex) throws IOException { + int code = 500; + String trace = ""; + if (ex instanceof SolrException) { + code = ((SolrException) ex).code(); + } + + // For any regular code, don't include the stack trace + if (code == 500 || code < 100) { + StringWriter sw = new StringWriter(); + ex.printStackTrace(new PrintWriter(sw)); + trace = "\n\n" + sw.toString(); + + SolrException.logOnce(log, null, ex); + + // non standard codes have undefined results with various servers + if (code < 100) { + log.warn("invalid return code: " + code); + code = 500; + } + } + res.sendError(code, ex.getMessage() + trace); + } + + // --------------------------------------------------------------------- + // --------------------------------------------------------------------- + + /** + * Set the prefix for all paths. This is useful if you want to apply the filter to something other then /*, perhaps + * because you are merging this filter into a larger web application. For example, if web.xml specifies: + * <filter-mapping> <filter-name>SolrRequestFilter</filter-name> <url-pattern>/xxx/*</url-pattern> </filter-mapping> + * Make sure to set the PathPrefix to "/xxx" either with this function or in web.xml. <init-param> + * <param-name>path-prefix</param-name> <param-value>/xxx</param-value> </init-param> + */ + public void setPathPrefix(String pathPrefix) { + this.pathPrefix = pathPrefix; + } + + public String getPathPrefix() { + return pathPrefix; + } +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrServiceImpl.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/impl/SolrServiceImpl.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,230 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.searchandindex.solr.impl; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Dictionary; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Hashtable; +import java.util.List; +import java.util.Map; + +import javax.servlet.Filter; + +import org.amdatu.core.config.templates.ConfigTemplateManager; +import org.amdatu.core.tenant.TenantEntity; +import org.amdatu.core.tenant.TenantException; +import org.amdatu.core.tenant.TenantManagementService; +import org.amdatu.searchandindex.solr.SolrService; +import org.amdatu.searchandindex.solr.osgi.Activator; +import org.amdatu.web.httpcontext.HttpContextServiceFactory; +import org.apache.felix.dm.Component; +import org.apache.felix.dm.DependencyManager; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; +import org.osgi.service.cm.ConfigurationException; +import org.osgi.service.cm.ManagedService; +import org.osgi.service.log.LogService; + +/** + * Implementation of the {@link SolrService} interface. + */ +public class SolrServiceImpl implements SolrService, ManagedService { + // Statics + private static final String SOLR_CONFIG_XML = "solr.xml"; + private static final String CONFIG_DIR = "conf"; + + // Services injected by the Felix dependency manager + private volatile LogService m_logService; + private volatile BundleContext m_bundleContext; + + private volatile DependencyManager m_dependencyManager; + private volatile HttpContextServiceFactory m_httpContextServiceFactory; + private volatile TenantManagementService m_tenantService; + private volatile ConfigTemplateManager m_configTemplateManager; + private volatile Component m_contextComponent; + + // Private members + private SolrDispatchFilter solrDispatchFilter = null; + private File m_workDir; + + /** + * The init() method is invoked by the Felix dependency manager. + */ + @SuppressWarnings("unchecked") + public void init() { + try { + m_logService.log(LogService.LOG_INFO, "Initializing Solr configuration"); + + // Create our own http context and register resources + m_contextComponent = m_httpContextServiceFactory.create(m_bundleContext, this); + + // Initialize storage configuration + // Load the URL of the storage-conf.xml and write it file using the config template + // manager, which automatically replaces configuration entries in that file + File storageConfigFile = new File(m_workDir, SOLR_CONFIG_XML); + m_workDir.mkdirs(); + Bundle bundle = m_bundleContext.getBundle(); + URL url = bundle.getResource(SOLR_CONFIG_XML); + try { + // Replace placeholders in the solr.xml config file + m_configTemplateManager.writeConfiguration(url, storageConfigFile); + + // Solr uses this system property to find its storage location. + System.setProperty("solr.solr.home", storageConfigFile.getParentFile().getAbsolutePath()); + + // Update the main config + File mainconfdir = new File(m_workDir, "/" + CONFIG_DIR); + mainconfdir.mkdirs(); + + // Find all entries in our 'conf' directory. + final Enumeration<URL> resources = bundle.findEntries(CONFIG_DIR, "*.*", true); + if (resources != null) { + while (resources.hasMoreElements()) { + final URL resource = resources.nextElement(); + File coreConfFile = new File(mainconfdir, resource.getFile().replace(CONFIG_DIR + "/", "")); + if (!coreConfFile.exists()) { + // Only write this file if it does not yet exist + m_configTemplateManager.writeConfiguration(resource, coreConfFile); + } + } + } + + // Update the core configs + List<String> cores = new ArrayList<String>(); + cores.add("Keyspace1/profile"); + cores.add("Keyspace2/profile"); + cores.add("Keyspace3/profile"); + + for (String core : cores) { + File coreconfdir = new File(m_workDir, core + "/conf"); + if (coreconfdir.exists()) { + coreconfdir.delete(); + } + coreconfdir.mkdirs(); + + // Find all entries in our 'conf' directory. + final Enumeration<URL> resources = bundle.findEntries("conf_core_profile", "*.*", true); + if (resources != null) { + while (resources.hasMoreElements()) { + final URL resource = resources.nextElement(); + File coreconffile = + new File(coreconfdir, resource.getFile().replaceAll("conf_core_profile/", "")); + if (!coreconffile.exists()) { + // Only write this file if it does not yet exist + m_configTemplateManager.writeConfiguration(resource, coreconffile); + } + } + } + + } + + } catch (IOException e) { + m_logService.log(LogService.LOG_ERROR, "Could not replace configuration entries in storage-conf.xml", e); + } + + } catch (Exception e) { + m_logService.log(LogService.LOG_ERROR, "Could not create context component", e); + } + } + + /** + * The start() method is invoked by the Felix dependency manager. + */ + public void start() { + m_logService.log(LogService.LOG_ERROR, getClass().getName() + " service Start"); + + // Create and register a http servlet filter. This filter is mapped on .* + Dictionary<String, String> filterProperties = new Hashtable<String, String>(); + filterProperties.put("pattern", "/" + Activator.RESOURCE_ID + "(|\\?.*|/.*)"); + filterProperties.put("service.ranking", "10"); + filterProperties.put("contextId", Activator.RESOURCE_ID); + + Map<String, String> tenantMap = new HashMap<String, String>(); + try { + for (TenantEntity tenant : m_tenantService.getTenants()) { + String tenanthostname = tenant.getProperties().get("hostname"); + if (tenanthostname != null) { + System.err.println("put: " + tenanthostname + " " + tenant.getId()); + tenantMap.put(tenanthostname, tenant.getId()); + } + } + } catch (TenantException e) { + e.printStackTrace(); + } + + solrDispatchFilter = + new SolrDispatchFilter("/" + Activator.RESOURCE_ID, m_workDir.getAbsolutePath(), tenantMap); + m_dependencyManager.add(m_dependencyManager.createComponent().setImplementation(solrDispatchFilter) + .setInterface(Filter.class.getName(), filterProperties)); + + } + + /** + * The stop() method is invoked by the Felix dependency manager. + */ + public void stop() { + if (solrDispatchFilter != null) { + solrDispatchFilter.destroy(); + } + + m_logService.log(LogService.LOG_ERROR, getClass().getName() + " service stopped"); + } + + /** + * The destroy() method is invoked by the Felix dependency manager. + */ + public void destroy() { + try { + if (m_contextComponent != null) { + m_contextComponent.stop(); + } + m_logService.log(LogService.LOG_ERROR, getClass().getName() + " service destroyed"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * {@inheritDoc} + */ + public URL getResource(String input) { + return null; + } + + /** + * {@inheritDoc} + */ + public String getResourceId() { + return Activator.RESOURCE_ID; + } + + @SuppressWarnings("unchecked") + public void updated(Dictionary dictionary) throws ConfigurationException { + if (dictionary != null) { + if (dictionary.get(CONFIG_WORKDIR) == null) { + throw new ConfigurationException("Missing configuration key", CONFIG_WORKDIR); + } + File workBaseDir = new File(System.getProperty("user.dir"), "work"); + m_workDir = new File(workBaseDir, (String) dictionary.get(CONFIG_WORKDIR)); + } + } +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/amdatu/searchandindex/solr/osgi/Activator.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,57 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.amdatu.searchandindex.solr.osgi; + +import org.amdatu.core.config.templates.ConfigTemplateManager; +import org.amdatu.core.tenant.TenantManagementService; +import org.amdatu.searchandindex.solr.SolrService; +import org.amdatu.searchandindex.solr.impl.SolrServiceImpl; +import org.amdatu.web.httpcontext.HttpContextServiceFactory; +import org.apache.felix.dm.DependencyActivatorBase; +import org.apache.felix.dm.DependencyManager; +import org.osgi.framework.BundleContext; +import org.osgi.service.http.HttpService; +import org.osgi.service.log.LogService; + +/** + * This is the OSGi activator for this Solr service bundle. + * + * @author ivol + */ +public class Activator extends DependencyActivatorBase { + /** + * The resource identifier for this bundle. Resources are only considered to be 'ours' when it is prefixed with this + * id. + */ + public static final String RESOURCE_ID = "index"; + + public void init(BundleContext context, DependencyManager manager) throws Exception { + // Create and register the Solr service. + manager.add(createComponent() + .setInterface(SolrService.class.getName(), null) + .setImplementation(SolrServiceImpl.class) + .add(createServiceDependency().setService(LogService.class).setRequired(true)) + .add(createServiceDependency().setService(HttpContextServiceFactory.class).setRequired(true)) + .add(createServiceDependency().setService(ConfigTemplateManager.class).setRequired(true)) + .add(createServiceDependency().setService(HttpService.class).setRequired(true)) + .add(createServiceDependency().setService(TenantManagementService.class).setRequired(true)) + .add(createConfigurationDependency().setPid(SolrService.PID))); + } + + public void destroy(BundleContext context, DependencyManager manager) throws Exception { + } +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrConfig.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrConfig.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,42 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.apache.solr.core; + +import java.io.IOException; +import java.io.InputStream; + +import javax.xml.parsers.ParserConfigurationException; + +import org.xml.sax.SAXException; + +public class NakamuraSolrConfig extends SolrConfig { + final SolrResourceLoader m_loader; + + public NakamuraSolrConfig(SolrResourceLoader loader, String name, InputStream is) + throws ParserConfigurationException, IOException, SAXException { + super(loader, name, is); + m_loader = loader; + } + + @Override + public SolrResourceLoader getResourceLoader() { + if (m_loader == null) { + return super.getResourceLoader(); + } + return m_loader; + } +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrResourceLoader.java ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/java/org/apache/solr/core/NakamuraSolrResourceLoader.java Mon Jan 24 16:22:35 2011 @@ -0,0 +1,36 @@ +/* + Copyright (C) 2010 Amdatu.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + */ +package org.apache.solr.core; + +import java.io.InputStream; + +public class NakamuraSolrResourceLoader extends SolrResourceLoader { + + public NakamuraSolrResourceLoader(String instanceDir, ClassLoader loader) { + super(instanceDir, loader); + } + + @Override + public InputStream openResource(String resource) { + final ClassLoader classLoader = this.getClass().getClassLoader(); + InputStream in = classLoader.getResourceAsStream(resource); + if (in == null) { + in = super.openResource(resource); + } + return in; + } +} Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/schema.xml ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/schema.xml Mon Jan 24 16:22:35 2011 @@ -0,0 +1,312 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + This is the Solr schema file. This file should be named "schema.xml" and + should be in the conf directory under the solr home + (i.e. ./solr/conf/schema.xml by default) + or located where the classloader for the Solr webapp can find it. + + This example schema is the recommended starting point for users. + It should be kept correct and concise, usable out-of-the-box. + + For more information, on how to customize this file, please see + http://wiki.apache.org/solr/SchemaXml + + PERFORMANCE NOTE: this schema includes many optional features and should not + be used for benchmarking. To improve performance one could + - set stored="false" for all fields possible (esp large fields) when you + only need to search on the field but don't need to return the original + value. + - set indexed="false" if you don't need to search on the field, but only + return the field as a result of searching on other indexed fields. + - remove all unneeded copyField statements + - for best index size and searching performance, set "index" to false + for all general text fields, use copyField to copy them to the + catchall "text" field, and use that for searching. + - For maximum indexing performance, use the StreamingUpdateSolrServer + java client. + - Remember to run the JVM in server mode, and use a higher logging level + that avoids logging every request +--> + +<schema name="blueconic" version="1.2"> + <!-- attribute "name" is the name of this schema and is only used for display purposes. + Applications should change this to reflect the nature of the search collection. + version="1.2" is Solr's version number for the schema syntax and semantics. It should + not normally be changed by applications. + 1.0: multiValued attribute did not exist, all fields are multiValued by nature + 1.1: multiValued attribute introduced, false by default + 1.2: omitTermFreqAndPositions attribute introduced, true by default except for text fields. + --> + + <types> + <!-- field type definitions. The "name" attribute is + just a label to be used by field definitions. The "class" + attribute and any other attributes determine the real + behavior of the fieldType. + Class names starting with "solr" refer to java classes in the + org.apache.solr.analysis package. + --> + + <!-- The StrField type is not analyzed, but indexed/stored verbatim. + - StrField and TextField support an optional compressThreshold which + limits compression (if enabled in the derived fields) to values which + exceed a certain size (in characters). + --> + <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/> + + <!-- boolean type: "true" or "false" --> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/> + <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings --> + <fieldtype name="binary" class="solr.BinaryField"/> + + <!-- The optional sortMissingLast and sortMissingFirst attributes are + currently supported on types that are sorted internally as strings. + This includes "string","boolean","sint","slong","sfloat","sdouble","pdate" + - If sortMissingLast="true", then a sort on this field will cause documents + without the field to come after documents with the field, + regardless of the requested sort order (asc or desc). + - If sortMissingFirst="true", then a sort on this field will cause documents + without the field to come before documents with the field, + regardless of the requested sort order. + - If sortMissingLast="false" and sortMissingFirst="false" (the default), + then default lucene sorting will be used which places docs without the + field first in an ascending sort and last in a descending sort. + --> + + <!-- + Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types. + --> + <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/> + + <!-- + Numeric field types that index each value at various levels of precision + to accelerate range queries when the number of values between the range + endpoints is large. See the javadoc for NumericRangeQuery for internal + implementation details. + + Smaller precisionStep values (specified in bits) will lead to more tokens + indexed per value, slightly larger index size, and faster range queries. + A precisionStep of 0 disables indexing at different precision levels. + --> + <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/> + + <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and + is a more restricted form of the canonical representation of dateTime + http://www.w3.org/TR/xmlschema-2/#dateTime + The trailing "Z" designates UTC time and is mandatory. + Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z + All other components are mandatory. + + Expressions can also be used to denote calculations that should be + performed relative to "NOW" to determine the value, ie... + + NOW/HOUR + ... Round to the start of the current hour + NOW-1DAY + ... Exactly 1 day prior to now + NOW/DAY+6MONTHS+3DAYS + ... 6 months and 3 days in the future from the start of + the current day + + Consult the DateField javadocs for more information. + + Note: For faster range queries, consider the tdate type + --> + <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/> + + <!-- A Trie based date field for faster date range queries and date faceting. --> + <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/> + + + <!-- + Note: + These should only be used for compatibility with existing indexes (created with older Solr versions) + or if "sortMissingFirst" or "sortMissingLast" functionality is needed. Use Trie based fields instead. + + Plain numeric field types that store and index the text + value verbatim (and hence don't support range queries, since the + lexicographic ordering isn't equal to the numeric ordering) + --> + <fieldType name="pint" class="solr.IntField" omitNorms="true"/> + <fieldType name="plong" class="solr.LongField" omitNorms="true"/> + <fieldType name="pfloat" class="solr.FloatField" omitNorms="true"/> + <fieldType name="pdouble" class="solr.DoubleField" omitNorms="true"/> + <fieldType name="pdate" class="solr.DateField" sortMissingLast="true" omitNorms="true"/> + + + <!-- + Note: + These should only be used for compatibility with existing indexes (created with older Solr versions) + or if "sortMissingFirst" or "sortMissingLast" functionality is needed. Use Trie based fields instead. + + Numeric field types that manipulate the value into + a string value that isn't human-readable in its internal form, + but with a lexicographic ordering the same as the numeric ordering, + so that range queries work correctly. + --> + <fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/> + <fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/> + + + <!-- The "RandomSortField" is not used to store or search any + data. You can declare fields of this type it in your schema + to generate pseudo-random orderings of your docs for sorting + purposes. The ordering is generated based on the field name + and the version of the index, As long as the index version + remains unchanged, and the same field name is reused, + the ordering of the docs will be consistent. + If you want different psuedo-random orderings of documents, + for the same version of the index, use a dynamicField and + change the name + --> + <fieldType name="random" class="solr.RandomSortField" indexed="true" /> + + <!-- solr.TextField allows the specification of custom text analyzers + specified as a tokenizer and a list of token filters. Different + analyzers may be specified for indexing and querying. + + The optional positionIncrementGap puts space between multiple fields of + this type on the same document, with the purpose of preventing false phrase + matching across fields. + + For more info on customizing your analyzer chain, please see + http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters + --> + + <!-- One can also specify an existing Analyzer class that has a + default constructor via the class attribute on the analyzer element + <fieldType name="text_greek" class="solr.TextField"> + <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/> + </fieldType> + --> + + <!-- A text field that only splits on whitespace for exact matching of words --> + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + </fieldType> + + <!-- A text field that uses WordDelimiterFilter to enable splitting and matching of + words on case-change, alpha numeric boundaries, and non-alphanumeric chars, + so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi". + Synonyms and stopwords are customized by external files, and stemming is enabled. + --> + <fieldType name="text" class="solr.TextField" positionIncrementGap="100"> + </fieldType> + + <!-- A general unstemmed text field - good if one does not know the language of the field --> + <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100"> + </fieldType> + + <!-- A general unstemmed text field that indexes tokens normally and also + reversed (via ReversedWildcardFilterFactory), to enable more efficient + leading wildcard queries. --> + <fieldType name="text_rev" class="solr.TextField" positionIncrementGap="100"> + </fieldType> + + <!-- since fields of this type are by default not stored or indexed, + any data added to them will be ignored outright. --> + <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" /> + </types> + + + <fields> + <!-- Valid attributes for fields: + name: mandatory - the name for the field + type: mandatory - the name of a previously defined type from the + <types> section + indexed: true if this field should be indexed (searchable or sortable) + stored: true if this field should be retrievable + compressed: [false] if this field should be stored using gzip compression + (this will only apply if the field type is compressable; among + the standard field types, only TextField and StrField are) + multiValued: true if this field may contain multiple values per document + omitNorms: (expert) set to true to omit the norms associated with + this field (this disables length normalization and index-time + boosting for the field, and saves some memory). Only full-text + fields or fields that need an index-time boost need norms. + termVectors: [false] set to true to store the term vector for a + given field. + When using MoreLikeThis, fields used for similarity should be + stored for best performance. + termPositions: Store position information with the term vector. + This will increase storage costs. + termOffsets: Store offset information with the term vector. This + will increase storage costs. + default: a value that should be used if no value is specified + when adding a document. + --> + + <field name="id" type="string" indexed="true" stored="true" required="true" /> + + <!-- catchall field, containing all other searchable text fields (implemented + via copyField further on in this schema --> + <field name="text" type="text" indexed="true" stored="false" multiValued="true"/> + + <field name="timestamp" type="date" indexed="true" stored="false" default="NOW" multiValued="false"/> + + <dynamicField name="*_i" type="sint" indexed="true" stored="false" omitNorms="true" multiValued="true"/> + <dynamicField name="*_s" type="string" indexed="true" stored="false" multiValued="true"/> + <dynamicField name="*_l" type="long" indexed="true" stored="false"/> + <dynamicField name="*_t" type="text" indexed="true" stored="false"/> + <dynamicField name="*_b" type="boolean" indexed="true" stored="false"/> + <dynamicField name="*_f" type="float" indexed="true" stored="false"/> + <dynamicField name="*_d" type="double" indexed="true" stored="false"/> + <dynamicField name="*_dt" type="date" indexed="true" stored="false"/> + + <!-- some trie-coded dynamic fields for faster range queries --> + <dynamicField name="*_ti" type="tint" indexed="true" stored="false"/> + <dynamicField name="*_tl" type="tlong" indexed="true" stored="false"/> + <dynamicField name="*_tf" type="tfloat" indexed="true" stored="false"/> + <dynamicField name="*_td" type="tdouble" indexed="true" stored="false"/> + <dynamicField name="*_tdt" type="tdate" indexed="true" stored="false"/> + + <dynamicField name="*_pi" type="pint" indexed="true" stored="false"/> + + <dynamicField name="ignored_*" type="ignored" multiValued="true"/> + <dynamicField name="attr_*" type="textgen" indexed="true" stored="false" multiValued="true"/> + + <dynamicField name="random_*" type="random" /> + + <!-- uncomment the following to ignore any fields that don't already match an existing + field name or dynamic field, rather than reporting them as an error. + alternately, change the type="ignored" to some other type e.g. "text" if you want + unknown fields indexed and/or stored by default --> + <!--dynamicField name="*" type="ignored" multiValued="true" /--> + + </fields> + + <!-- Field to use to determine and enforce document uniqueness. + Unless this field is marked with required="false", it will be a required field + --> + <uniqueKey>id</uniqueKey> + + <!-- field for the QueryParser to use when an explicit fieldname is absent --> + <defaultSearchField>text</defaultSearchField> + + <!-- SolrQueryParser configuration: defaultOperator="AND|OR" --> + <solrQueryParser defaultOperator="OR"/> +</schema> Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/solrconfig.xml ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/conf_core_profile/solrconfig.xml Mon Jan 24 16:22:35 2011 @@ -0,0 +1,61 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + http://wiki.apache.org/solr/SolrConfigXml + +--> + +<config> + <indexDefaults> + <!-- Sets the amount of RAM that may be used by Lucene indexing + for buffering added documents and deletions before they are + flushed to the Directory. --> + <ramBufferSizeMB>32</ramBufferSizeMB> + <lockType>simple</lockType> + <!-- + Expert: + Controls how often Lucene loads terms into memory --> + <!--<termIndexInterval>256</termIndexInterval>--> + </indexDefaults> + + <jmx /> + + <query> + <maxBooleanClauses>102400</maxBooleanClauses> + </query> + + <updateHandler class="solr.DirectUpdateHandler2" /> + + <requestDispatcher handleSelect="true" > + <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" /> + </requestDispatcher> + + <requestHandler name="standard" class="solr.StandardRequestHandler" default="true"> + <arr name="last-components"> + </arr> + </requestHandler> + + <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" /> + <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" /> + + <!-- config for the admin interface --> + <admin> + <defaultQuery>solr</defaultQuery> + </admin> + +</config> + Added: sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/solr.xml ============================================================================== --- (empty file) +++ sandbox/ivol/amdatu-searchandindex/solr/src/main/resources/solr.xml Mon Jan 24 16:22:35 2011 @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8" ?> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version + 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 Unless required by + applicable law or agreed to in writing, software distributed under the + License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the + License. + --> + + <!-- + All (relative) paths are relative to the installation path persistent: + Save changes made via the API to this file sharedLib: path to a lib + directory that will be shared across all cores + --> + +<solr persistent="true"> + <!-- + adminPath: RequestHandler path to manage cores. If 'null' (or absent), + cores will not be manageable via request handler + --> + <cores adminPath="/admin/cores"> + <core name="Keyspace1_profile" instanceDir="Keyspace1/profile" /> + <core name="Keyspace2_profile" instanceDir="Keyspace2/profile" /> + <core name="Keyspace3_profile" instanceDir="Keyspace3/profile" /> + </cores> +</solr> \ No newline at end of file
