http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/config/schemas/solr/nexustiles/conf/solrconfig.xml ---------------------------------------------------------------------- diff --git a/data-access/config/schemas/solr/nexustiles/conf/solrconfig.xml b/data-access/config/schemas/solr/nexustiles/conf/solrconfig.xml new file mode 100644 index 0000000..1765625 --- /dev/null +++ b/data-access/config/schemas/solr/nexustiles/conf/solrconfig.xml @@ -0,0 +1,1408 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + For more details about configurations options that may appear in + this file, see http://wiki.apache.org/solr/SolrConfigXml. +--> +<config> + <!-- In all configuration below, a prefix of "solr." for class names + is an alias that causes solr to search appropriate packages, + including org.apache.solr.(search|update|request|core|analysis) + + You may also specify a fully qualified Java classname if you + have your own custom plugins. + --> + + <!-- Controls what version of Lucene various components of Solr + adhere to. Generally, you want to use the latest version to + get all bug fixes and improvements. It is highly recommended + that you fully re-index after changing this setting as it can + affect both how text is indexed and queried. + --> + <luceneMatchVersion>6.4.2</luceneMatchVersion> + + <!-- <lib/> directives can be used to instruct Solr to load any Jars + identified and use them to resolve any "plugins" specified in + your solrconfig.xml or schema.xml (ie: Analyzers, Request + Handlers, etc...). + + All directories and paths are resolved relative to the + instanceDir. + + Please note that <lib/> directives are processed in the order + that they appear in your solrconfig.xml file, and are "stacked" + on top of each other when building a ClassLoader - so if you have + plugin jars with dependencies on other jars, the "lower level" + dependency jars should be loaded first. + + If a "./lib" directory exists in your instanceDir, all files + found in it are included as if you had used the following + syntax... + + <lib dir="./lib" /> + --> + + <!-- A 'dir' option by itself adds any files found in the directory + to the classpath, this is useful for including all jars in a + directory. + + When a 'regex' is specified in addition to a 'dir', only the + files in that directory which completely match the regex + (anchored on both ends) will be included. + + If a 'dir' option (with or without a regex) is used and nothing + is found that matches, a warning will be logged. + + The examples below can be used to load some solr-contribs along + with their external dependencies. + --> + <lib dir="${solr.install.dir:../../../..}/contrib/extraction/lib" regex=".*\.jar" /> + <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-cell-\d.*\.jar" /> + + <lib dir="${solr.install.dir:../../../..}/contrib/clustering/lib/" regex=".*\.jar" /> + <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-clustering-\d.*\.jar" /> + + <lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" /> + <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" /> + + <lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" /> + <lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-velocity-\d.*\.jar" /> + <!-- an exact 'path' can be used instead of a 'dir' to specify a + specific jar file. This will cause a serious error to be logged + if it can't be loaded. + --> + <!-- + <lib path="../a-jar-that-does-not-exist.jar" /> + --> + + <!-- Data Directory + + Used to specify an alternate directory to hold all index data + other than the default ./data under the Solr home. If + replication is in use, this should match the replication + configuration. + --> + <dataDir>${solr.data.dir:}</dataDir> + + + <!-- The DirectoryFactory to use for indexes. + + solr.StandardDirectoryFactory is filesystem + based and tries to pick the best implementation for the current + JVM and platform. solr.NRTCachingDirectoryFactory, the default, + wraps solr.StandardDirectoryFactory and caches small files in memory + for better NRT performance. + + One can force a particular implementation via solr.MMapDirectoryFactory, + solr.NIOFSDirectoryFactory, or solr.SimpleFSDirectoryFactory. + + solr.RAMDirectoryFactory is memory based, not + persistent, and doesn't work with replication. + --> + <directoryFactory name="DirectoryFactory" + class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/> + + <!-- The CodecFactory for defining the format of the inverted index. + The default implementation is SchemaCodecFactory, which is the official Lucene + index format, but hooks into the schema to provide per-field customization of + the postings lists and per-document values in the fieldType element + (postingsFormat/docValuesFormat). Note that most of the alternative implementations + are experimental, so if you choose to customize the index format, it's a good + idea to convert back to the official format e.g. via IndexWriter.addIndexes(IndexReader) + before upgrading to a newer version to avoid unnecessary reindexing. + A "compressionMode" string element can be added to <codecFactory> to choose + between the existing compression modes in the default codec: "BEST_SPEED" (default) + or "BEST_COMPRESSION". + --> + <codecFactory class="solr.SchemaCodecFactory"/> + + <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Index Config - These settings control low-level behavior of indexing + Most example settings here show the default value, but are commented + out, to more easily see where customizations have been made. + + Note: This replaces <indexDefaults> and <mainIndex> from older versions + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --> + <indexConfig> + <!-- maxFieldLength was removed in 4.0. To get similar behavior, include a + LimitTokenCountFilterFactory in your fieldType definition. E.g. + <filter class="solr.LimitTokenCountFilterFactory" maxTokenCount="10000"/> + --> + <!-- Maximum time to wait for a write lock (ms) for an IndexWriter. Default: 1000 --> + <!-- <writeLockTimeout>1000</writeLockTimeout> --> + + <!-- Expert: Enabling compound file will use less files for the index, + using fewer file descriptors on the expense of performance decrease. + Default in Lucene is "true". Default in Solr is "false" (since 3.6) --> + <!-- <useCompoundFile>false</useCompoundFile> --> + + <!-- ramBufferSizeMB sets the amount of RAM that may be used by Lucene + indexing for buffering added documents and deletions before they are + flushed to the Directory. + maxBufferedDocs sets a limit on the number of documents buffered + before flushing. + If both ramBufferSizeMB and maxBufferedDocs is set, then + Lucene will flush based on whichever limit is hit first. --> + <!-- <ramBufferSizeMB>100</ramBufferSizeMB> --> + <!-- <maxBufferedDocs>1000</maxBufferedDocs> --> + + <!-- Expert: Merge Policy + The Merge Policy in Lucene controls how merging of segments is done. + The default since Solr/Lucene 3.3 is TieredMergePolicy. + The default since Lucene 2.3 was the LogByteSizeMergePolicy, + Even older versions of Lucene used LogDocMergePolicy. + --> + <!-- + <mergePolicyFactory class="org.apache.solr.index.TieredMergePolicyFactory"> + <int name="maxMergeAtOnce">10</int> + <int name="segmentsPerTier">10</int> + <double name="noCFSRatio">0.1</double> + </mergePolicyFactory> + --> + + <!-- Expert: Merge Scheduler + The Merge Scheduler in Lucene controls how merges are + performed. The ConcurrentMergeScheduler (Lucene 2.3 default) + can perform merges in the background using separate threads. + The SerialMergeScheduler (Lucene 2.2 default) does not. + --> + <!-- + <mergeScheduler class="org.apache.lucene.index.ConcurrentMergeScheduler"/> + --> + + <!-- LockFactory + + This option specifies which Lucene LockFactory implementation + to use. + + single = SingleInstanceLockFactory - suggested for a + read-only index or when there is no possibility of + another process trying to modify the index. + native = NativeFSLockFactory - uses OS native file locking. + Do not use when multiple solr webapps in the same + JVM are attempting to share a single index. + simple = SimpleFSLockFactory - uses a plain file for locking + + Defaults: 'native' is default for Solr3.6 and later, otherwise + 'simple' is the default + + More details on the nuances of each LockFactory... + http://wiki.apache.org/lucene-java/AvailableLockFactories + --> + <lockType>${solr.lock.type:native}</lockType> + + <!-- Commit Deletion Policy + Custom deletion policies can be specified here. The class must + implement org.apache.lucene.index.IndexDeletionPolicy. + + The default Solr IndexDeletionPolicy implementation supports + deleting index commit points on number of commits, age of + commit point and optimized status. + + The latest commit point should always be preserved regardless + of the criteria. + --> + <!-- + <deletionPolicy class="solr.SolrDeletionPolicy"> + --> + <!-- The number of commit points to be kept --> + <!-- <str name="maxCommitsToKeep">1</str> --> + <!-- The number of optimized commit points to be kept --> + <!-- <str name="maxOptimizedCommitsToKeep">0</str> --> + <!-- + Delete all commit points once they have reached the given age. + Supports DateMathParser syntax e.g. + --> + <!-- + <str name="maxCommitAge">30MINUTES</str> + <str name="maxCommitAge">1DAY</str> + --> + <!-- + </deletionPolicy> + --> + + <!-- Lucene Infostream + + To aid in advanced debugging, Lucene provides an "InfoStream" + of detailed information when indexing. + + Setting The value to true will instruct the underlying Lucene + IndexWriter to write its debugging info the specified file + --> + <!-- <infoStream file="INFOSTREAM.txt">false</infoStream> --> + </indexConfig> + + + <!-- JMX + + This example enables JMX if and only if an existing MBeanServer + is found, use this if you want to configure JMX through JVM + parameters. Remove this to disable exposing Solr configuration + and statistics to JMX. + + For more details see http://wiki.apache.org/solr/SolrJmx + --> + <jmx /> + <!-- If you want to connect to a particular server, specify the + agentId + --> + <!-- <jmx agentId="myAgent" /> --> + <!-- If you want to start a new MBeanServer, specify the serviceUrl --> + <!-- <jmx serviceUrl="service:jmx:rmi:///jndi/rmi://localhost:9999/solr"/> + --> + + <!-- The default high-performance update handler --> + <updateHandler class="solr.DirectUpdateHandler2"> + + <!-- Enables a transaction log, used for real-time get, durability, and + and solr cloud replica recovery. The log can grow as big as + uncommitted changes to the index, so use of a hard autoCommit + is recommended (see below). + "dir" - the target directory for transaction logs, defaults to the + solr data directory. + "numVersionBuckets" - sets the number of buckets used to keep + track of max version values when checking for re-ordered + updates; increase this value to reduce the cost of + synchronizing access to version buckets during high-volume + indexing, this requires 8 bytes (long) * numVersionBuckets + of heap space per Solr core. + --> + <updateLog> + <str name="dir">${solr.ulog.dir:}</str> + <int name="numVersionBuckets">${solr.ulog.numVersionBuckets:65536}</int> + </updateLog> + + <!-- AutoCommit + + Perform a hard commit automatically under certain conditions. + Instead of enabling autoCommit, consider using "commitWithin" + when adding documents. + + http://wiki.apache.org/solr/UpdateXmlMessages + + maxDocs - Maximum number of documents to add since the last + commit before automatically triggering a new commit. + + maxTime - Maximum amount of time in ms that is allowed to pass + since a document was added before automatically + triggering a new commit. + openSearcher - if false, the commit causes recent index changes + to be flushed to stable storage, but does not cause a new + searcher to be opened to make those changes visible. + + If the updateLog is enabled, then it's highly recommended to + have some sort of hard autoCommit to limit the log size. + --> + <autoCommit> + <maxTime>${solr.autoCommit.maxTime:15000}</maxTime> + <openSearcher>false</openSearcher> + </autoCommit> + + <!-- softAutoCommit is like autoCommit except it causes a + 'soft' commit which only ensures that changes are visible + but does not ensure that data is synced to disk. This is + faster and more near-realtime friendly than a hard commit. + --> + + <autoSoftCommit> + <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime> + </autoSoftCommit> + + <!-- Update Related Event Listeners + + Various IndexWriter related events can trigger Listeners to + take actions. + + postCommit - fired after every commit or optimize command + postOptimize - fired after every optimize command + --> + <!-- The RunExecutableListener executes an external command from a + hook such as postCommit or postOptimize. + + exe - the name of the executable to run + dir - dir to use as the current working directory. (default=".") + wait - the calling thread waits until the executable returns. + (default="true") + args - the arguments to pass to the program. (default is none) + env - environment variables to set. (default is none) + --> + <!-- This example shows how RunExecutableListener could be used + with the script based replication... + http://wiki.apache.org/solr/CollectionDistribution + --> + <!-- + <listener event="postCommit" class="solr.RunExecutableListener"> + <str name="exe">solr/bin/snapshooter</str> + <str name="dir">.</str> + <bool name="wait">true</bool> + <arr name="args"> <str>arg1</str> <str>arg2</str> </arr> + <arr name="env"> <str>MYVAR=val1</str> </arr> + </listener> + --> + + </updateHandler> + + <!-- IndexReaderFactory + + Use the following format to specify a custom IndexReaderFactory, + which allows for alternate IndexReader implementations. + + ** Experimental Feature ** + + Please note - Using a custom IndexReaderFactory may prevent + certain other features from working. The API to + IndexReaderFactory may change without warning or may even be + removed from future releases if the problems cannot be + resolved. + + + ** Features that may not work with custom IndexReaderFactory ** + + The ReplicationHandler assumes a disk-resident index. Using a + custom IndexReader implementation may cause incompatibility + with ReplicationHandler and may cause replication to not work + correctly. See SOLR-1366 for details. + + --> + <!-- + <indexReaderFactory name="IndexReaderFactory" class="package.class"> + <str name="someArg">Some Value</str> + </indexReaderFactory > + --> + + <!-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Query section - these settings control query time things like caches + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --> + <query> + <!-- Max Boolean Clauses + + Maximum number of clauses in each BooleanQuery, an exception + is thrown if exceeded. + + ** WARNING ** + + This option actually modifies a global Lucene property that + will affect all SolrCores. If multiple solrconfig.xml files + disagree on this property, the value at any given moment will + be based on the last SolrCore to be initialized. + + --> + <maxBooleanClauses>1024</maxBooleanClauses> + + + <!-- Solr Internal Query Caches + + There are two implementations of cache available for Solr, + LRUCache, based on a synchronized LinkedHashMap, and + FastLRUCache, based on a ConcurrentHashMap. + + FastLRUCache has faster gets and slower puts in single + threaded operation and thus is generally faster than LRUCache + when the hit ratio of the cache is high (> 75%), and may be + faster under other scenarios on multi-cpu systems. + --> + + <!-- Filter Cache + + Cache used by SolrIndexSearcher for filters (DocSets), + unordered sets of *all* documents that match a query. When a + new searcher is opened, its caches may be prepopulated or + "autowarmed" using data from caches in the old searcher. + autowarmCount is the number of items to prepopulate. For + LRUCache, the autowarmed items will be the most recently + accessed items. + + Parameters: + class - the SolrCache implementation LRUCache or + (LRUCache or FastLRUCache) + size - the maximum number of entries in the cache + initialSize - the initial capacity (number of entries) of + the cache. (see java.util.HashMap) + autowarmCount - the number of entries to prepopulate from + and old cache. + maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed + to occupy. Note that when this option is specified, the size + and initialSize parameters are ignored. + --> + <filterCache class="solr.FastLRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + + <!-- Query Result Cache + + Caches results of searches - ordered lists of document ids + (DocList) based on a query, a sort, and the range of documents requested. + Additional supported parameter by LRUCache: + maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed + to occupy + --> + <queryResultCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + + <!-- Document Cache + + Caches Lucene Document objects (the stored fields for each + document). Since Lucene internal document ids are transient, + this cache will not be autowarmed. + --> + <documentCache class="solr.LRUCache" + size="512" + initialSize="512" + autowarmCount="0"/> + + <!-- custom cache currently used by block join --> + <cache name="perSegFilter" + class="solr.search.LRUCache" + size="10" + initialSize="0" + autowarmCount="10" + regenerator="solr.NoOpRegenerator" /> + + <!-- Field Value Cache + + Cache used to hold field values that are quickly accessible + by document id. The fieldValueCache is created by default + even if not configured here. + --> + <!-- + <fieldValueCache class="solr.FastLRUCache" + size="512" + autowarmCount="128" + showItems="32" /> + --> + + <!-- Custom Cache + + Example of a generic cache. These caches may be accessed by + name through SolrIndexSearcher.getCache(),cacheLookup(), and + cacheInsert(). The purpose is to enable easy caching of + user/application level data. The regenerator argument should + be specified as an implementation of solr.CacheRegenerator + if autowarming is desired. + --> + <!-- + <cache name="myUserCache" + class="solr.LRUCache" + size="4096" + initialSize="1024" + autowarmCount="1024" + regenerator="com.mycompany.MyRegenerator" + /> + --> + + + <!-- Lazy Field Loading + + If true, stored fields that are not requested will be loaded + lazily. This can result in a significant speed improvement + if the usual case is to not load all stored fields, + especially if the skipped fields are large compressed text + fields. + --> + <enableLazyFieldLoading>true</enableLazyFieldLoading> + + <!-- Use Filter For Sorted Query + + A possible optimization that attempts to use a filter to + satisfy a search. If the requested sort does not include + score, then the filterCache will be checked for a filter + matching the query. If found, the filter will be used as the + source of document ids, and then the sort will be applied to + that. + + For most situations, this will not be useful unless you + frequently get the same search repeatedly with different sort + options, and none of them ever use "score" + --> + <!-- + <useFilterForSortedQuery>true</useFilterForSortedQuery> + --> + + <!-- Result Window Size + + An optimization for use with the queryResultCache. When a search + is requested, a superset of the requested number of document ids + are collected. For example, if a search for a particular query + requests matching documents 10 through 19, and queryWindowSize is 50, + then documents 0 through 49 will be collected and cached. Any further + requests in that range can be satisfied via the cache. + --> + <queryResultWindowSize>20</queryResultWindowSize> + + <!-- Maximum number of documents to cache for any entry in the + queryResultCache. + --> + <queryResultMaxDocsCached>200</queryResultMaxDocsCached> + + <!-- Query Related Event Listeners + + Various IndexSearcher related events can trigger Listeners to + take actions. + + newSearcher - fired whenever a new searcher is being prepared + and there is a current searcher handling requests (aka + registered). It can be used to prime certain caches to + prevent long request times for certain requests. + + firstSearcher - fired whenever a new searcher is being + prepared but there is no current registered searcher to handle + requests or to gain autowarming data from. + + + --> + <!-- QuerySenderListener takes an array of NamedList and executes a + local query request for each NamedList in sequence. + --> + <listener event="newSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <!-- + <lst><str name="q">solr</str><str name="sort">price asc</str></lst> + <lst><str name="q">rocks</str><str name="sort">weight asc</str></lst> + --> + </arr> + </listener> + <listener event="firstSearcher" class="solr.QuerySenderListener"> + <arr name="queries"> + <!-- + <lst> + <str name="q">static firstSearcher warming in solrconfig.xml</str> + </lst> + --> + </arr> + </listener> + + <!-- Use Cold Searcher + + If a search request comes in and there is no current + registered searcher, then immediately register the still + warming searcher and use it. If "false" then all requests + will block until the first searcher is done warming. + --> + <useColdSearcher>false</useColdSearcher> + + </query> + + + <!-- Request Dispatcher + + This section contains instructions for how the SolrDispatchFilter + should behave when processing requests for this SolrCore. + + handleSelect is a legacy option that affects the behavior of requests + such as /select?qt=XXX + + handleSelect="true" will cause the SolrDispatchFilter to process + the request and dispatch the query to a handler specified by the + "qt" param, assuming "/select" isn't already registered. + + handleSelect="false" will cause the SolrDispatchFilter to + ignore "/select" requests, resulting in a 404 unless a handler + is explicitly registered with the name "/select" + + handleSelect="true" is not recommended for new users, but is the default + for backwards compatibility + --> + <requestDispatcher handleSelect="false" > + <!-- Request Parsing + + These settings indicate how Solr Requests may be parsed, and + what restrictions may be placed on the ContentStreams from + those requests + + enableRemoteStreaming - enables use of the stream.file + and stream.url parameters for specifying remote streams. + + multipartUploadLimitInKB - specifies the max size (in KiB) of + Multipart File Uploads that Solr will allow in a Request. + + formdataUploadLimitInKB - specifies the max size (in KiB) of + form data (application/x-www-form-urlencoded) sent via + POST. You can use POST to pass request parameters not + fitting into the URL. + + addHttpRequestToContext - if set to true, it will instruct + the requestParsers to include the original HttpServletRequest + object in the context map of the SolrQueryRequest under the + key "httpRequest". It will not be used by any of the existing + Solr components, but may be useful when developing custom + plugins. + + *** WARNING *** + The settings below authorize Solr to fetch remote files, You + should make sure your system has some authentication before + using enableRemoteStreaming="true" + + --> + <requestParsers enableRemoteStreaming="true" + multipartUploadLimitInKB="2048000" + formdataUploadLimitInKB="2048" + addHttpRequestToContext="false"/> + + <!-- HTTP Caching + + Set HTTP caching related parameters (for proxy caches and clients). + + The options below instruct Solr not to output any HTTP Caching + related headers + --> + <httpCaching never304="true" /> + <!-- If you include a <cacheControl> directive, it will be used to + generate a Cache-Control header (as well as an Expires header + if the value contains "max-age=") + + By default, no Cache-Control header is generated. + + You can use the <cacheControl> option even if you have set + never304="true" + --> + <!-- + <httpCaching never304="true" > + <cacheControl>max-age=30, public</cacheControl> + </httpCaching> + --> + <!-- To enable Solr to respond with automatically generated HTTP + Caching headers, and to response to Cache Validation requests + correctly, set the value of never304="false" + + This will cause Solr to generate Last-Modified and ETag + headers based on the properties of the Index. + + The following options can also be specified to affect the + values of these headers... + + lastModFrom - the default value is "openTime" which means the + Last-Modified value (and validation against If-Modified-Since + requests) will all be relative to when the current Searcher + was opened. You can change it to lastModFrom="dirLastMod" if + you want the value to exactly correspond to when the physical + index was last modified. + + etagSeed="..." is an option you can change to force the ETag + header (and validation against If-None-Match requests) to be + different even if the index has not changed (ie: when making + significant changes to your config file) + + (lastModifiedFrom and etagSeed are both ignored if you use + the never304="true" option) + --> + <!-- + <httpCaching lastModifiedFrom="openTime" + etagSeed="Solr"> + <cacheControl>max-age=30, public</cacheControl> + </httpCaching> + --> + </requestDispatcher> + + <!-- Request Handlers + + http://wiki.apache.org/solr/SolrRequestHandler + + Incoming queries will be dispatched to a specific handler by name + based on the path specified in the request. + + Legacy behavior: If the request path uses "/select" but no Request + Handler has that name, and if handleSelect="true" has been specified in + the requestDispatcher, then the Request Handler is dispatched based on + the qt parameter. Handlers without a leading '/' are accessed this way + like so: http://host/app/[core/]select?qt=name If no qt is + given, then the requestHandler that declares default="true" will be + used or the one named "standard". + + If a Request Handler is declared with startup="lazy", then it will + not be initialized until the first request that uses it. + + --> + <!-- SearchHandler + + http://wiki.apache.org/solr/SearchHandler + + For processing Search Queries, the primary Request Handler + provided with Solr is "SearchHandler" It delegates to a sequent + of SearchComponents (see below) and supports distributed + queries across multiple shards + --> + <requestHandler name="/select" class="solr.SearchHandler"> + <!-- default values for query parameters can be specified, these + will be overridden by parameters in the request + --> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <int name="rows">10</int> + <!-- <str name="df">text</str> --> + </lst> + <!-- In addition to defaults, "appends" params can be specified + to identify values which should be appended to the list of + multi-val params from the query (or the existing "defaults"). + --> + <!-- In this example, the param "fq=instock:true" would be appended to + any query time fq params the user may specify, as a mechanism for + partitioning the index, independent of any user selected filtering + that may also be desired (perhaps as a result of faceted searching). + + NOTE: there is *absolutely* nothing a client can do to prevent these + "appends" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <!-- + <lst name="appends"> + <str name="fq">inStock:true</str> + </lst> + --> + <!-- "invariants" are a way of letting the Solr maintainer lock down + the options available to Solr clients. Any params values + specified here are used regardless of what values may be specified + in either the query, the "defaults", or the "appends" params. + + In this example, the facet.field and facet.query params would + be fixed, limiting the facets clients can use. Faceting is + not turned on by default - but if the client does specify + facet=true in the request, these are the only facets they + will be able to see counts for; regardless of what other + facet.field or facet.query params they may specify. + + NOTE: there is *absolutely* nothing a client can do to prevent these + "invariants" values from being used, so don't use this mechanism + unless you are sure you always want it. + --> + <!-- + <lst name="invariants"> + <str name="facet.field">cat</str> + <str name="facet.field">manu_exact</str> + <str name="facet.query">price:[* TO 500]</str> + <str name="facet.query">price:[500 TO *]</str> + </lst> + --> + <!-- If the default list of SearchComponents is not desired, that + list can either be overridden completely, or components can be + prepended or appended to the default list. (see below) + --> + <!-- + <arr name="components"> + <str>nameOfCustomComponent1</str> + <str>nameOfCustomComponent2</str> + </arr> + --> + </requestHandler> + + <!-- A request handler that returns indented JSON by default --> + <requestHandler name="/query" class="solr.SearchHandler"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + <str name="wt">json</str> + <str name="indent">true</str> + </lst> + </requestHandler> + + + <!-- A Robust Example + + This example SearchHandler declaration shows off usage of the + SearchHandler with many defaults declared + + Note that multiple instances of the same Request Handler + (SearchHandler) can be registered multiple times with different + names (and different init parameters) + --> + <requestHandler name="/browse" class="solr.SearchHandler" useParams="query,facets,velocity,browse"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + </lst> + </requestHandler> + + <initParams path="/update/**,/query,/select,/tvrh,/elevate,/spell,/browse"> + <lst name="defaults"> + <str name="df">_text_</str> + </lst> + </initParams> + + <initParams path="/update/**"> + <lst name="defaults"> + <str name="update.chain">add-unknown-fields-to-the-schema</str> + </lst> + </initParams> + + <!-- Solr Cell Update Request Handler + + http://wiki.apache.org/solr/ExtractingRequestHandler + + --> + <requestHandler name="/update/extract" + startup="lazy" + class="solr.extraction.ExtractingRequestHandler" > + <lst name="defaults"> + <str name="lowernames">true</str> + <str name="fmap.meta">ignored_</str> + <str name="fmap.content">_text_</str> + </lst> + </requestHandler> + + <!-- Search Components + + Search components are registered to SolrCore and used by + instances of SearchHandler (which can access them by name) + + By default, the following components are available: + + <searchComponent name="query" class="solr.QueryComponent" /> + <searchComponent name="facet" class="solr.FacetComponent" /> + <searchComponent name="mlt" class="solr.MoreLikeThisComponent" /> + <searchComponent name="highlight" class="solr.HighlightComponent" /> + <searchComponent name="stats" class="solr.StatsComponent" /> + <searchComponent name="debug" class="solr.DebugComponent" /> + + Default configuration in a requestHandler would look like: + + <arr name="components"> + <str>query</str> + <str>facet</str> + <str>mlt</str> + <str>highlight</str> + <str>stats</str> + <str>debug</str> + </arr> + + If you register a searchComponent to one of the standard names, + that will be used instead of the default. + + To insert components before or after the 'standard' components, use: + + <arr name="first-components"> + <str>myFirstComponentName</str> + </arr> + + <arr name="last-components"> + <str>myLastComponentName</str> + </arr> + + NOTE: The component registered with the name "debug" will + always be executed after the "last-components" + + --> + + <!-- Spell Check + + The spell check component can return a list of alternative spelling + suggestions. + + http://wiki.apache.org/solr/SpellCheckComponent + --> + <searchComponent name="spellcheck" class="solr.SpellCheckComponent"> + + <str name="queryAnalyzerFieldType">text_general</str> + + <!-- Multiple "Spell Checkers" can be declared and used by this + component + --> + + <!-- a spellchecker built from a field of the main index --> + <lst name="spellchecker"> + <str name="name">default</str> + <str name="field">_text_</str> + <str name="classname">solr.DirectSolrSpellChecker</str> + <!-- the spellcheck distance measure used, the default is the internal levenshtein --> + <str name="distanceMeasure">internal</str> + <!-- minimum accuracy needed to be considered a valid spellcheck suggestion --> + <float name="accuracy">0.5</float> + <!-- the maximum #edits we consider when enumerating terms: can be 1 or 2 --> + <int name="maxEdits">2</int> + <!-- the minimum shared prefix when enumerating terms --> + <int name="minPrefix">1</int> + <!-- maximum number of inspections per result. --> + <int name="maxInspections">5</int> + <!-- minimum length of a query term to be considered for correction --> + <int name="minQueryLength">4</int> + <!-- maximum threshold of documents a query term can appear to be considered for correction --> + <float name="maxQueryFrequency">0.01</float> + <!-- uncomment this to require suggestions to occur in 1% of the documents + <float name="thresholdTokenFrequency">.01</float> + --> + </lst> + + <!-- a spellchecker that can break or combine words. See "/spell" handler below for usage --> + <!-- + <lst name="spellchecker"> + <str name="name">wordbreak</str> + <str name="classname">solr.WordBreakSolrSpellChecker</str> + <str name="field">name</str> + <str name="combineWords">true</str> + <str name="breakWords">true</str> + <int name="maxChanges">10</int> + </lst> + --> + </searchComponent> + + <!-- A request handler for demonstrating the spellcheck component. + + NOTE: This is purely as an example. The whole purpose of the + SpellCheckComponent is to hook it into the request handler that + handles your normal user queries so that a separate request is + not needed to get suggestions. + + IN OTHER WORDS, THERE IS REALLY GOOD CHANCE THE SETUP BELOW IS + NOT WHAT YOU WANT FOR YOUR PRODUCTION SYSTEM! + + See http://wiki.apache.org/solr/SpellCheckComponent for details + on the request parameters. + --> + <requestHandler name="/spell" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <!-- Solr will use suggestions from both the 'default' spellchecker + and from the 'wordbreak' spellchecker and combine them. + collations (re-written queries) can include a combination of + corrections from both spellcheckers --> + <str name="spellcheck.dictionary">default</str> + <str name="spellcheck">on</str> + <str name="spellcheck.extendedResults">true</str> + <str name="spellcheck.count">10</str> + <str name="spellcheck.alternativeTermCount">5</str> + <str name="spellcheck.maxResultsForSuggest">5</str> + <str name="spellcheck.collate">true</str> + <str name="spellcheck.collateExtendedResults">true</str> + <str name="spellcheck.maxCollationTries">10</str> + <str name="spellcheck.maxCollations">5</str> + </lst> + <arr name="last-components"> + <str>spellcheck</str> + </arr> + </requestHandler> + + <!-- Term Vector Component + + http://wiki.apache.org/solr/TermVectorComponent + --> + <searchComponent name="tvComponent" class="solr.TermVectorComponent"/> + + <!-- A request handler for demonstrating the term vector component + + This is purely as an example. + + In reality you will likely want to add the component to your + already specified request handlers. + --> + <requestHandler name="/tvrh" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <bool name="tv">true</bool> + </lst> + <arr name="last-components"> + <str>tvComponent</str> + </arr> + </requestHandler> + + <!-- Clustering Component. (Omitted here. See the default Solr example for a typical configuration.) --> + + <!-- Terms Component + + http://wiki.apache.org/solr/TermsComponent + + A component to return terms and document frequency of those + terms + --> + <searchComponent name="terms" class="solr.TermsComponent"/> + + <!-- A request handler for demonstrating the terms component --> + <requestHandler name="/terms" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <bool name="terms">true</bool> + <bool name="distrib">false</bool> + </lst> + <arr name="components"> + <str>terms</str> + </arr> + </requestHandler> + + + <!-- Query Elevation Component + + http://wiki.apache.org/solr/QueryElevationComponent + + a search component that enables you to configure the top + results for a given query regardless of the normal lucene + scoring. + --> + <searchComponent name="elevator" class="solr.QueryElevationComponent" > + <!-- pick a fieldType to analyze queries --> + <str name="queryFieldType">string</str> + <str name="config-file">elevate.xml</str> + </searchComponent> + + <!-- A request handler for demonstrating the elevator component --> + <requestHandler name="/elevate" class="solr.SearchHandler" startup="lazy"> + <lst name="defaults"> + <str name="echoParams">explicit</str> + </lst> + <arr name="last-components"> + <str>elevator</str> + </arr> + </requestHandler> + + <!-- Highlighting Component + + http://wiki.apache.org/solr/HighlightingParameters + --> + <searchComponent class="solr.HighlightComponent" name="highlight"> + <highlighting> + <!-- Configure the standard fragmenter --> + <!-- This could most likely be commented out in the "default" case --> + <fragmenter name="gap" + default="true" + class="solr.highlight.GapFragmenter"> + <lst name="defaults"> + <int name="hl.fragsize">100</int> + </lst> + </fragmenter> + + <!-- A regular-expression-based fragmenter + (for sentence extraction) + --> + <fragmenter name="regex" + class="solr.highlight.RegexFragmenter"> + <lst name="defaults"> + <!-- slightly smaller fragsizes work better because of slop --> + <int name="hl.fragsize">70</int> + <!-- allow 50% slop on fragment sizes --> + <float name="hl.regex.slop">0.5</float> + <!-- a basic sentence pattern --> + <str name="hl.regex.pattern">[-\w ,/\n\"']{20,200}</str> + </lst> + </fragmenter> + + <!-- Configure the standard formatter --> + <formatter name="html" + default="true" + class="solr.highlight.HtmlFormatter"> + <lst name="defaults"> + <str name="hl.simple.pre"><![CDATA[<em>]]></str> + <str name="hl.simple.post"><![CDATA[</em>]]></str> + </lst> + </formatter> + + <!-- Configure the standard encoder --> + <encoder name="html" + class="solr.highlight.HtmlEncoder" /> + + <!-- Configure the standard fragListBuilder --> + <fragListBuilder name="simple" + class="solr.highlight.SimpleFragListBuilder"/> + + <!-- Configure the single fragListBuilder --> + <fragListBuilder name="single" + class="solr.highlight.SingleFragListBuilder"/> + + <!-- Configure the weighted fragListBuilder --> + <fragListBuilder name="weighted" + default="true" + class="solr.highlight.WeightedFragListBuilder"/> + + <!-- default tag FragmentsBuilder --> + <fragmentsBuilder name="default" + default="true" + class="solr.highlight.ScoreOrderFragmentsBuilder"> + <!-- + <lst name="defaults"> + <str name="hl.multiValuedSeparatorChar">/</str> + </lst> + --> + </fragmentsBuilder> + + <!-- multi-colored tag FragmentsBuilder --> + <fragmentsBuilder name="colored" + class="solr.highlight.ScoreOrderFragmentsBuilder"> + <lst name="defaults"> + <str name="hl.tag.pre"><![CDATA[ + <b style="background:yellow">,<b style="background:lawgreen">, + <b style="background:aquamarine">,<b style="background:magenta">, + <b style="background:palegreen">,<b style="background:coral">, + <b style="background:wheat">,<b style="background:khaki">, + <b style="background:lime">,<b style="background:deepskyblue">]]></str> + <str name="hl.tag.post"><![CDATA[</b>]]></str> + </lst> + </fragmentsBuilder> + + <boundaryScanner name="default" + default="true" + class="solr.highlight.SimpleBoundaryScanner"> + <lst name="defaults"> + <str name="hl.bs.maxScan">10</str> + <str name="hl.bs.chars">.,!? 	 </str> + </lst> + </boundaryScanner> + + <boundaryScanner name="breakIterator" + class="solr.highlight.BreakIteratorBoundaryScanner"> + <lst name="defaults"> + <!-- type should be one of CHARACTER, WORD(default), LINE and SENTENCE --> + <str name="hl.bs.type">WORD</str> + <!-- language and country are used when constructing Locale object. --> + <!-- And the Locale object will be used when getting instance of BreakIterator --> + <str name="hl.bs.language">en</str> + <str name="hl.bs.country">US</str> + </lst> + </boundaryScanner> + </highlighting> + </searchComponent> + + <!-- Update Processors + + Chains of Update Processor Factories for dealing with Update + Requests can be declared, and then used by name in Update + Request Processors + + http://wiki.apache.org/solr/UpdateRequestProcessor + + --> + + <!-- Add unknown fields to the schema + + An example field type guessing update processor that will + attempt to parse string-typed field values as Booleans, Longs, + Doubles, or Dates, and then add schema fields with the guessed + field types. + + This requires that the schema is both managed and mutable, by + declaring schemaFactory as ManagedIndexSchemaFactory, with + mutable specified as true. + + See http://wiki.apache.org/solr/GuessingFieldTypes + --> + <updateRequestProcessorChain name="add-unknown-fields-to-the-schema"> + <!-- UUIDUpdateProcessorFactory will generate an id if none is present in the incoming document --> + <processor class="solr.UUIDUpdateProcessorFactory" /> + <processor class="solr.RemoveBlankFieldUpdateProcessorFactory"/> + <processor class="solr.FieldNameMutatingUpdateProcessorFactory"> + <str name="pattern">[^\w-\.]</str> + <str name="replacement">_</str> + </processor> + <processor class="solr.ParseBooleanFieldUpdateProcessorFactory"/> + <processor class="solr.ParseLongFieldUpdateProcessorFactory"/> + <processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/> + <processor class="solr.ParseDateFieldUpdateProcessorFactory"> + <arr name="format"> + <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str> + <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str> + <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str> + <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str> + <str>yyyy-MM-dd'T'HH:mm:ssZ</str> + <str>yyyy-MM-dd'T'HH:mm:ss</str> + <str>yyyy-MM-dd'T'HH:mmZ</str> + <str>yyyy-MM-dd'T'HH:mm</str> + <str>yyyy-MM-dd HH:mm:ss.SSSZ</str> + <str>yyyy-MM-dd HH:mm:ss,SSSZ</str> + <str>yyyy-MM-dd HH:mm:ss.SSS</str> + <str>yyyy-MM-dd HH:mm:ss,SSS</str> + <str>yyyy-MM-dd HH:mm:ssZ</str> + <str>yyyy-MM-dd HH:mm:ss</str> + <str>yyyy-MM-dd HH:mmZ</str> + <str>yyyy-MM-dd HH:mm</str> + <str>yyyy-MM-dd</str> + </arr> + </processor> + <processor class="solr.AddSchemaFieldsUpdateProcessorFactory"> + <str name="defaultFieldType">strings</str> + <lst name="typeMapping"> + <str name="valueClass">java.lang.Boolean</str> + <str name="fieldType">booleans</str> + </lst> + <lst name="typeMapping"> + <str name="valueClass">java.util.Date</str> + <str name="fieldType">tdates</str> + </lst> + <lst name="typeMapping"> + <str name="valueClass">java.lang.Long</str> + <str name="valueClass">java.lang.Integer</str> + <str name="fieldType">tlongs</str> + </lst> + <lst name="typeMapping"> + <str name="valueClass">java.lang.Number</str> + <str name="fieldType">tdoubles</str> + </lst> + </processor> + <processor class="solr.LogUpdateProcessorFactory"/> + <processor class="solr.DistributedUpdateProcessorFactory"/> + <processor class="solr.RunUpdateProcessorFactory"/> + </updateRequestProcessorChain> + + <!-- Deduplication + + An example dedup update processor that creates the "id" field + on the fly based on the hash code of some other fields. This + example has overwriteDupes set to false since we are using the + id field as the signatureField and Solr will maintain + uniqueness based on that anyway. + + --> + <!-- + <updateRequestProcessorChain name="dedupe"> + <processor class="solr.processor.SignatureUpdateProcessorFactory"> + <bool name="enabled">true</bool> + <str name="signatureField">id</str> + <bool name="overwriteDupes">false</bool> + <str name="fields">name,features,cat</str> + <str name="signatureClass">solr.processor.Lookup3Signature</str> + </processor> + <processor class="solr.LogUpdateProcessorFactory" /> + <processor class="solr.RunUpdateProcessorFactory" /> + </updateRequestProcessorChain> + --> + + <!-- Language identification + + This example update chain identifies the language of the incoming + documents using the langid contrib. The detected language is + written to field language_s. No field name mapping is done. + The fields used for detection are text, title, subject and description, + making this example suitable for detecting languages form full-text + rich documents injected via ExtractingRequestHandler. + See more about langId at http://wiki.apache.org/solr/LanguageDetection + --> + <!-- + <updateRequestProcessorChain name="langid"> + <processor class="org.apache.solr.update.processor.TikaLanguageIdentifierUpdateProcessorFactory"> + <str name="langid.fl">text,title,subject,description</str> + <str name="langid.langField">language_s</str> + <str name="langid.fallback">en</str> + </processor> + <processor class="solr.LogUpdateProcessorFactory" /> + <processor class="solr.RunUpdateProcessorFactory" /> + </updateRequestProcessorChain> + --> + + <!-- Script update processor + + This example hooks in an update processor implemented using JavaScript. + + See more about the script update processor at http://wiki.apache.org/solr/ScriptUpdateProcessor + --> + <!-- + <updateRequestProcessorChain name="script"> + <processor class="solr.StatelessScriptUpdateProcessorFactory"> + <str name="script">update-script.js</str> + <lst name="params"> + <str name="config_param">example config parameter</str> + </lst> + </processor> + <processor class="solr.RunUpdateProcessorFactory" /> + </updateRequestProcessorChain> + --> + + <!-- Response Writers + + http://wiki.apache.org/solr/QueryResponseWriter + + Request responses will be written using the writer specified by + the 'wt' request parameter matching the name of a registered + writer. + + The "default" writer is the default and will be used if 'wt' is + not specified in the request. + --> + <!-- The following response writers are implicitly configured unless + overridden... + --> + <!-- + <queryResponseWriter name="xml" + default="true" + class="solr.XMLResponseWriter" /> + <queryResponseWriter name="json" class="solr.JSONResponseWriter"/> + <queryResponseWriter name="python" class="solr.PythonResponseWriter"/> + <queryResponseWriter name="ruby" class="solr.RubyResponseWriter"/> + <queryResponseWriter name="php" class="solr.PHPResponseWriter"/> + <queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/> + <queryResponseWriter name="csv" class="solr.CSVResponseWriter"/> + <queryResponseWriter name="schema.xml" class="solr.SchemaXmlResponseWriter"/> + --> + + <queryResponseWriter name="json" class="solr.JSONResponseWriter"> + <!-- For the purposes of the tutorial, JSON responses are written as + plain text so that they are easy to read in *any* browser. + If you expect a MIME type of "application/json" just remove this override. + --> + <str name="content-type">text/plain; charset=UTF-8</str> + </queryResponseWriter> + + <!-- + Custom response writers can be declared as needed... + --> + <queryResponseWriter name="velocity" class="solr.VelocityResponseWriter" startup="lazy"> + <str name="template.base.dir">${velocity.template.base.dir:}</str> + <str name="solr.resource.loader.enabled">${velocity.solr.resource.loader.enabled:true}</str> + <str name="params.resource.loader.enabled">${velocity.params.resource.loader.enabled:false}</str> + </queryResponseWriter> + + <!-- XSLT response writer transforms the XML output by any xslt file found + in Solr's conf/xslt directory. Changes to xslt files are checked for + every xsltCacheLifetimeSeconds. + --> + <queryResponseWriter name="xslt" class="solr.XSLTResponseWriter"> + <int name="xsltCacheLifetimeSeconds">5</int> + </queryResponseWriter> + + <!-- Query Parsers + + https://cwiki.apache.org/confluence/display/solr/Query+Syntax+and+Parsing + + Multiple QParserPlugins can be registered by name, and then + used in either the "defType" param for the QueryComponent (used + by SearchHandler) or in LocalParams + --> + <!-- example of registering a query parser --> + <!-- + <queryParser name="myparser" class="com.mycompany.MyQParserPlugin"/> + --> + + <!-- Function Parsers + + http://wiki.apache.org/solr/FunctionQuery + + Multiple ValueSourceParsers can be registered by name, and then + used as function names when using the "func" QParser. + --> + <!-- example of registering a custom function parser --> + <!-- + <valueSourceParser name="myfunc" + class="com.mycompany.MyValueSourceParser" /> + --> + + + <!-- Document Transformers + http://wiki.apache.org/solr/DocTransformers + --> + <!-- + Could be something like: + <transformer name="db" class="com.mycompany.LoadFromDatabaseTransformer" > + <int name="connection">jdbc://....</int> + </transformer> + + To add a constant value to all docs, use: + <transformer name="mytrans2" class="org.apache.solr.response.transform.ValueAugmenterFactory" > + <int name="value">5</int> + </transformer> + + If you want the user to still be able to change it with _value:something_ use this: + <transformer name="mytrans3" class="org.apache.solr.response.transform.ValueAugmenterFactory" > + <double name="defaultValue">5</double> + </transformer> + + If you are using the QueryElevationComponent, you may wish to mark documents that get boosted. The + EditorialMarkerFactory will do exactly that: + <transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" /> + --> +</config>
http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/config/schemas/solr/nexustiles/conf/stopwords.txt ---------------------------------------------------------------------- diff --git a/data-access/config/schemas/solr/nexustiles/conf/stopwords.txt b/data-access/config/schemas/solr/nexustiles/conf/stopwords.txt new file mode 100644 index 0000000..ae1e83e --- /dev/null +++ b/data-access/config/schemas/solr/nexustiles/conf/stopwords.txt @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/config/schemas/solr/nexustiles/conf/synonyms.txt ---------------------------------------------------------------------- diff --git a/data-access/config/schemas/solr/nexustiles/conf/synonyms.txt b/data-access/config/schemas/solr/nexustiles/conf/synonyms.txt new file mode 100644 index 0000000..7f72128 --- /dev/null +++ b/data-access/config/schemas/solr/nexustiles/conf/synonyms.txt @@ -0,0 +1,29 @@ +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#----------------------------------------------------------------------- +#some test synonym mappings unlikely to appear in real input text +aaafoo => aaabar +bbbfoo => bbbfoo bbbbar +cccfoo => cccbar cccbaz +fooaaa,baraaa,bazaaa + +# Some synonym groups specific to this example +GB,gib,gigabyte,gigabytes +MB,mib,megabyte,megabytes +Television, Televisions, TV, TVs +#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming +#after us won't split it into two words. + +# Synonym mappings can be used for spelling correction too +pixima => pixma + http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/config/schemas/solr/nexustiles/core.properties ---------------------------------------------------------------------- diff --git a/data-access/config/schemas/solr/nexustiles/core.properties b/data-access/config/schemas/solr/nexustiles/core.properties new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/nexustiles/__init__.py ---------------------------------------------------------------------- diff --git a/data-access/nexustiles/__init__.py b/data-access/nexustiles/__init__.py new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/nexustiles/config/datastores.ini ---------------------------------------------------------------------- diff --git a/data-access/nexustiles/config/datastores.ini b/data-access/nexustiles/config/datastores.ini new file mode 100644 index 0000000..194760c --- /dev/null +++ b/data-access/nexustiles/config/datastores.ini @@ -0,0 +1,9 @@ +[cassandra] +host=127.0.0.1 +keyspace=nexustiles +local_datacenter=datacenter1 +protocol_version=3 + +[solr] +host=localhost:8983 +core=nexustiles \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/nexustiles/dao/CassandraProxy.pyx ---------------------------------------------------------------------- diff --git a/data-access/nexustiles/dao/CassandraProxy.pyx b/data-access/nexustiles/dao/CassandraProxy.pyx new file mode 100644 index 0000000..8b005f5 --- /dev/null +++ b/data-access/nexustiles/dao/CassandraProxy.pyx @@ -0,0 +1,146 @@ +import uuid +from ConfigParser import NoOptionError +from multiprocessing.synchronize import Lock + +import nexusproto.NexusContent_pb2 as nexusproto +import numpy as np +from cassandra.cqlengine import columns, connection, CQLEngineException +from cassandra.cqlengine.models import Model +from cassandra.policies import TokenAwarePolicy, DCAwareRoundRobinPolicy +from nexusproto.serialization import from_shaped_array + +INIT_LOCK = Lock() + + +class NexusTileData(Model): + __table_name__ = 'sea_surface_temp' + tile_id = columns.UUID(primary_key=True) + tile_blob = columns.Blob() + + __nexus_tile = None + + def _get_nexus_tile(self): + if self.__nexus_tile is None: + self.__nexus_tile = nexusproto.TileData.FromString(self.tile_blob) + + return self.__nexus_tile + + def get_raw_data_array(self): + + nexus_tile = self._get_nexus_tile() + the_tile_type = nexus_tile.tile.WhichOneof("tile_type") + + the_tile_data = getattr(nexus_tile.tile, the_tile_type) + + return from_shaped_array(the_tile_data.variable_data) + + def get_lat_lon_time_data_meta(self): + if self._get_nexus_tile().HasField('grid_tile'): + grid_tile = self._get_nexus_tile().grid_tile + + grid_tile_data = np.ma.masked_invalid(from_shaped_array(grid_tile.variable_data)) + latitude_data = np.ma.masked_invalid(from_shaped_array(grid_tile.latitude)) + longitude_data = np.ma.masked_invalid(from_shaped_array(grid_tile.longitude)) + + if len(grid_tile_data.shape) == 2: + grid_tile_data = grid_tile_data[np.newaxis, :] + + # Extract the meta data + meta_data = {} + for meta_data_obj in grid_tile.meta_data: + name = meta_data_obj.name + meta_array = np.ma.masked_invalid(from_shaped_array(meta_data_obj.meta_data)) + if len(meta_array.shape) == 2: + meta_array = meta_array[np.newaxis, :] + meta_data[name] = meta_array + + return latitude_data, longitude_data, np.array([grid_tile.time]), grid_tile_data, meta_data + elif self._get_nexus_tile().HasField('swath_tile'): + swath_tile = self._get_nexus_tile().swath_tile + + latitude_data = np.ma.masked_invalid(from_shaped_array(swath_tile.latitude)).reshape(-1) + longitude_data = np.ma.masked_invalid(from_shaped_array(swath_tile.longitude)).reshape(-1) + time_data = np.ma.masked_invalid(from_shaped_array(swath_tile.time)).reshape(-1) + + # Simplify the tile if the time dimension is the same value repeated + if np.all(time_data == np.min(time_data)): + time_data = np.array([np.min(time_data)]) + + swath_tile_data = np.ma.masked_invalid(from_shaped_array(swath_tile.variable_data)) + + tile_data = self._to_standard_index(swath_tile_data, + (len(time_data), len(latitude_data), len(longitude_data))) + + # Extract the meta data + meta_data = {} + for meta_data_obj in swath_tile.meta_data: + name = meta_data_obj.name + actual_meta_array = np.ma.masked_invalid(from_shaped_array(meta_data_obj.meta_data)) + reshaped_meta_array = self._to_standard_index(actual_meta_array, tile_data.shape) + meta_data[name] = reshaped_meta_array + + return latitude_data, longitude_data, time_data, tile_data, meta_data + else: + raise NotImplementedError("Only supports grid_tile and swath_tile") + + @staticmethod + def _to_standard_index(data_array, desired_shape): + + if desired_shape[0] == 1: + reshaped_array = np.ma.masked_all((desired_shape[1], desired_shape[2])) + row, col = np.indices(data_array.shape) + + reshaped_array[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array[ + row.flat, col.flat] + reshaped_array.mask[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array.mask[ + row.flat, col.flat] + reshaped_array = reshaped_array[np.newaxis, :] + else: + reshaped_array = np.ma.masked_all(desired_shape) + row, col = np.indices(data_array.shape) + + reshaped_array[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array[ + row.flat, col.flat] + reshaped_array.mask[np.diag_indices(desired_shape[1], len(reshaped_array.shape))] = data_array.mask[ + row.flat, col.flat] + + return reshaped_array + + +class CassandraProxy(object): + def __init__(self, config): + self.config = config + self.__cass_url = config.get("cassandra", "host") + self.__cass_keyspace = config.get("cassandra", "keyspace") + self.__cass_local_DC = config.get("cassandra", "local_datacenter") + self.__cass_protocol_version = config.getint("cassandra", "protocol_version") + try: + self.__cass_port = config.getint("cassandra", "port") + except NoOptionError: + self.__cass_port = 9042 + + with INIT_LOCK: + try: + connection.get_cluster() + except CQLEngineException: + self.__open() + + def __open(self): + + dc_policy = DCAwareRoundRobinPolicy(self.__cass_local_DC) + token_policy = TokenAwarePolicy(dc_policy) + connection.setup([host for host in self.__cass_url.split(',')], self.__cass_keyspace, + protocol_version=self.__cass_protocol_version, load_balancing_policy=token_policy, + port=self.__cass_port) + + def fetch_nexus_tiles(self, *tile_ids): + tile_ids = [uuid.UUID(str(tile_id)) for tile_id in tile_ids if + (isinstance(tile_id, str) or isinstance(tile_id, unicode))] + + res = [] + for tile_id in tile_ids: + filterResults = NexusTileData.objects.filter(tile_id=tile_id) + if len(filterResults) > 0: + res.append(filterResults[0]) + + return res http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/nexustiles/dao/SolrProxy.pyx ---------------------------------------------------------------------- diff --git a/data-access/nexustiles/dao/SolrProxy.pyx b/data-access/nexustiles/dao/SolrProxy.pyx new file mode 100644 index 0000000..0d775e6 --- /dev/null +++ b/data-access/nexustiles/dao/SolrProxy.pyx @@ -0,0 +1,567 @@ +import json +import logging +import threading +import time +from datetime import datetime + +import requests +import solr +from shapely import wkt + +SOLR_CON_LOCK = threading.Lock() +thread_local = threading.local() + + +class SolrProxy(object): + def __init__(self, config): + self.solrUrl = config.get("solr", "host") + self.solrCore = config.get("solr", "core") + self.logger = logging.getLogger('nexus') + + with SOLR_CON_LOCK: + solrcon = getattr(thread_local, 'solrcon', None) + if solrcon is None: + solrcon = solr.Solr('http://%s/solr/%s' % (self.solrUrl, self.solrCore), debug=False) + thread_local.solrcon = solrcon + + self.solrcon = solrcon + + def find_tile_by_id(self, tile_id): + + search = 'id:%s' % tile_id + + params = { + 'rows': 1 + } + + results, start, found = self.do_query(*(search, None, None, True, None), **params) + + assert len(results) == 1, "Found %s results, expected exactly 1" % len(results) + return [results[0]] + + def find_tiles_by_id(self, tile_ids, ds=None, **kwargs): + + if ds is not None: + search = 'dataset_s:%s' % ds + else: + search = '*:*' + + additionalparams = { + 'fq': [ + "{!terms f=id}%s" % ','.join(tile_ids) + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + results = self.do_query_all(*(search, None, None, False, None), **additionalparams) + + assert len(results) == len(tile_ids), "Found %s results, expected exactly %s" % (len(results), len(tile_ids)) + return results + + def find_min_date_from_tiles(self, tile_ids, ds=None, **kwargs): + + if ds is not None: + search = 'dataset_s:%s' % ds + else: + search = '*:*' + + kwargs['rows'] = 1 + kwargs['fl'] = 'tile_min_time_dt' + kwargs['sort'] = ['tile_min_time_dt asc'] + additionalparams = { + 'fq': [ + "{!terms f=id}%s" % ','.join(tile_ids) if len(tile_ids) > 0 else '' + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + results, start, found = self.do_query(*(search, None, None, True, None), **additionalparams) + + return results[0]['tile_min_time_dt'] + + def find_max_date_from_tiles(self, tile_ids, ds=None, **kwargs): + + if ds is not None: + search = 'dataset_s:%s' % ds + else: + search = '*:*' + + kwargs['rows'] = 1 + kwargs['fl'] = 'tile_max_time_dt' + kwargs['sort'] = ['tile_max_time_dt desc'] + additionalparams = { + 'fq': [ + "{!terms f=id}%s" % ','.join(tile_ids) if len(tile_ids) > 0 else '' + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + results, start, found = self.do_query(*(search, None, None, True, None), **additionalparams) + + return results[0]['tile_max_time_dt'] + + def get_data_series_list(self): + + datasets = self.get_data_series_list_simple() + + for dataset in datasets: + dataset['start'] = time.mktime(self.find_min_date_from_tiles([], ds=dataset['title']).timetuple()) * 1000 + dataset['end'] = time.mktime(self.find_max_date_from_tiles([], ds=dataset['title']).timetuple()) * 1000 + + return datasets + + def get_data_series_list_simple(self): + search = "*:*" + params = { + 'rows': 0, + "facet": "true", + "facet.field": "dataset_s", + "facet.mincount": "1" + } + + response = self.do_query_raw(*(search, None, None, False, None), **params) + l = [] + for g, v in response.facet_counts["facet_fields"]["dataset_s"].items(): + l.append({ + "shortName": g, + "title": g, + "tileCount": v + }) + l = sorted(l, key=lambda entry: entry["title"]) + return l + + def find_tile_by_polygon_and_most_recent_day_of_year(self, bounding_polygon, ds, day_of_year): + + search = 'dataset_s:%s' % ds + + params = { + 'fq': [ + "{!field f=geo}Intersects(%s)" % bounding_polygon.wkt, + "tile_count_i:[1 TO *]", + "day_of_year_i:[* TO %s]" % day_of_year + ], + 'rows': 1 + } + + results, start, found = self.do_query( + *(search, None, None, True, ('day_of_year_i desc',)), **params) + + return [results[0]] + + def find_days_in_range_asc(self, min_lat, max_lat, min_lon, max_lon, ds, start_time, end_time, **kwargs): + + search = 'dataset_s:%s' % ds + + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + additionalparams = { + 'fq': [ + "geo:[%s,%s TO %s,%s]" % (min_lat, min_lon, max_lat, max_lon), + "{!frange l=0 u=0}ms(tile_min_time_dt,tile_max_time_dt)", + "tile_count_i:[1 TO *]", + "tile_min_time_dt:[%s TO %s] " % (search_start_s, search_end_s) + ], + 'rows': 0, + 'facet': 'true', + 'facet_field': 'tile_min_time_dt', + 'facet_mincount': '1', + 'facet_limit': '-1' + } + + self._merge_kwargs(additionalparams, **kwargs) + + response = self.do_query_raw(*(search, None, None, False, None), **additionalparams) + + daysinrangeasc = sorted( + [(datetime.strptime(a_date, '%Y-%m-%dT%H:%M:%SZ') - datetime.utcfromtimestamp(0)).total_seconds() for a_date + in response.facet_counts['facet_fields']['tile_min_time_dt'].keys()]) + + return daysinrangeasc + + def find_all_tiles_in_box_sorttimeasc(self, min_lat, max_lat, min_lon, max_lon, ds, start_time=0, + end_time=-1, **kwargs): + + search = 'dataset_s:%s' % ds + + additionalparams = { + 'fq': [ + "geo:[%s,%s TO %s,%s]" % (min_lat, min_lon, max_lat, max_lon), + "tile_count_i:[1 TO *]" + ] + } + + if 0 < start_time <= end_time: + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + time_clause = "(" \ + "tile_min_time_dt:[%s TO %s] " \ + "OR tile_max_time_dt:[%s TO %s] " \ + "OR (tile_min_time_dt:[* TO %s] AND tile_max_time_dt:[%s TO *])" \ + ")" % ( + search_start_s, search_end_s, + search_start_s, search_end_s, + search_start_s, search_end_s + ) + additionalparams['fq'].append(time_clause) + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all( + *(search, None, None, False, 'tile_min_time_dt asc, tile_max_time_dt asc'), + **additionalparams) + + def find_all_tiles_in_polygon_sorttimeasc(self, bounding_polygon, ds, start_time=0, end_time=-1, **kwargs): + + search = 'dataset_s:%s' % ds + + additionalparams = { + 'fq': [ + "{!field f=geo}Intersects(%s)" % bounding_polygon.wkt, + "tile_count_i:[1 TO *]" + ] + } + + if 0 < start_time <= end_time: + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + time_clause = "(" \ + "tile_min_time_dt:[%s TO %s] " \ + "OR tile_max_time_dt:[%s TO %s] " \ + "OR (tile_min_time_dt:[* TO %s] AND tile_max_time_dt:[%s TO *])" \ + ")" % ( + search_start_s, search_end_s, + search_start_s, search_end_s, + search_start_s, search_end_s + ) + additionalparams['fq'].append(time_clause) + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all( + *(search, None, None, False, 'tile_min_time_dt asc, tile_max_time_dt asc'), + **additionalparams) + + def find_all_tiles_in_polygon(self, bounding_polygon, ds, start_time=0, end_time=-1, **kwargs): + + search = 'dataset_s:%s' % ds + + additionalparams = { + 'fq': [ + "{!field f=geo}Intersects(%s)" % bounding_polygon.wkt, + "tile_count_i:[1 TO *]" + ] + } + + if 0 < start_time <= end_time: + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + time_clause = "(" \ + "tile_min_time_dt:[%s TO %s] " \ + "OR tile_max_time_dt:[%s TO %s] " \ + "OR (tile_min_time_dt:[* TO %s] AND tile_max_time_dt:[%s TO *])" \ + ")" % ( + search_start_s, search_end_s, + search_start_s, search_end_s, + search_start_s, search_end_s + ) + additionalparams['fq'].append(time_clause) + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all( + *(search, None, None, False, None), + **additionalparams) + + def find_distinct_bounding_boxes_in_polygon(self, bounding_polygon, ds, start_time=0, end_time=-1, **kwargs): + + search = 'dataset_s:%s' % ds + + additionalparams = { + 'fq': [ + "{!field f=geo}Intersects(%s)" % bounding_polygon.wkt, + "tile_count_i:[1 TO *]" + ], + 'rows': 0, + 'facet': 'true', + 'facet.field': 'geo_s', + 'facet.limit': -1, + 'facet.mincount': 1 + } + + if 0 < start_time <= end_time: + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + time_clause = "(" \ + "tile_min_time_dt:[%s TO %s] " \ + "OR tile_max_time_dt:[%s TO %s] " \ + "OR (tile_min_time_dt:[* TO %s] AND tile_max_time_dt:[%s TO *])" \ + ")" % ( + search_start_s, search_end_s, + search_start_s, search_end_s, + search_start_s, search_end_s + ) + additionalparams['fq'].append(time_clause) + + self._merge_kwargs(additionalparams, **kwargs) + + response = self.do_query_raw(*(search, None, None, False, None), **additionalparams) + + distinct_bounds = [wkt.loads(key).bounds for key in response.facet_counts["facet_fields"]["geo_s"].keys()] + + return distinct_bounds + + def find_tiles_by_exact_bounds(self, minx, miny, maxx, maxy, ds, start_time=0, end_time=-1, **kwargs): + + search = 'dataset_s:%s' % ds + + additionalparams = { + 'fq': [ + "tile_min_lon:\"%s\"" % minx, + "tile_min_lat:\"%s\"" % miny, + "tile_max_lon:\"%s\"" % maxx, + "tile_max_lat:\"%s\"" % maxy, + "tile_count_i:[1 TO *]" + ] + } + + if 0 < start_time <= end_time: + search_start_s = datetime.utcfromtimestamp(start_time).strftime('%Y-%m-%dT%H:%M:%SZ') + search_end_s = datetime.utcfromtimestamp(end_time).strftime('%Y-%m-%dT%H:%M:%SZ') + + time_clause = "(" \ + "tile_min_time_dt:[%s TO %s] " \ + "OR tile_max_time_dt:[%s TO %s] " \ + "OR (tile_min_time_dt:[* TO %s] AND tile_max_time_dt:[%s TO *])" \ + ")" % ( + search_start_s, search_end_s, + search_start_s, search_end_s, + search_start_s, search_end_s + ) + additionalparams['fq'].append(time_clause) + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all( + *(search, None, None, False, None), + **additionalparams) + + def find_all_tiles_in_box_at_time(self, min_lat, max_lat, min_lon, max_lon, ds, search_time, **kwargs): + search = 'dataset_s:%s' % ds + + the_time = datetime.utcfromtimestamp(search_time).strftime('%Y-%m-%dT%H:%M:%SZ') + time_clause = "(" \ + "tile_min_time_dt:[* TO %s] " \ + "AND tile_max_time_dt:[%s TO *] " \ + ")" % ( + the_time, the_time + ) + + additionalparams = { + 'fq': [ + "geo:[%s,%s TO %s,%s]" % (min_lat, min_lon, max_lat, max_lon), + "tile_count_i:[1 TO *]", + time_clause + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all(*(search, None, None, False, None), **additionalparams) + + def find_all_tiles_in_polygon_at_time(self, bounding_polygon, ds, search_time, **kwargs): + search = 'dataset_s:%s' % ds + + the_time = datetime.utcfromtimestamp(search_time).strftime('%Y-%m-%dT%H:%M:%SZ') + time_clause = "(" \ + "tile_min_time_dt:[* TO %s] " \ + "AND tile_max_time_dt:[%s TO *] " \ + ")" % ( + the_time, the_time + ) + + additionalparams = { + 'fq': [ + "{!field f=geo}Intersects(%s)" % bounding_polygon.wkt, + "tile_count_i:[1 TO *]", + time_clause + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all(*(search, None, None, False, None), **additionalparams) + + def find_all_tiles_within_box_at_time(self, min_lat, max_lat, min_lon, max_lon, ds, time, **kwargs): + search = 'dataset_s:%s' % ds + + the_time = datetime.utcfromtimestamp(time).strftime('%Y-%m-%dT%H:%M:%SZ') + time_clause = "(" \ + "tile_min_time_dt:[* TO %s] " \ + "AND tile_max_time_dt:[%s TO *] " \ + ")" % ( + the_time, the_time + ) + + additionalparams = { + 'fq': [ + "geo:\"Within(ENVELOPE(%s,%s,%s,%s))\"" % (min_lon, max_lon, max_lat, min_lat), + "tile_count_i:[1 TO *]", + time_clause + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all(*(search, "product(tile_avg_val_d, tile_count_i),*", None, False, None), + **additionalparams) + + def find_all_boundary_tiles_at_time(self, min_lat, max_lat, min_lon, max_lon, ds, time, **kwargs): + search = 'dataset_s:%s' % ds + + the_time = datetime.utcfromtimestamp(time).strftime('%Y-%m-%dT%H:%M:%SZ') + time_clause = "(" \ + "tile_min_time_dt:[* TO %s] " \ + "AND tile_max_time_dt:[%s TO *] " \ + ")" % ( + the_time, the_time + ) + + additionalparams = { + 'fq': [ + "geo:\"Intersects(MultiLineString((%s %s, %s %s),(%s %s, %s %s),(%s %s, %s %s),(%s %s, %s %s)))\"" % ( + min_lon, max_lat, max_lon, max_lat, min_lon, max_lat, min_lon, min_lat, max_lon, max_lat, max_lon, + min_lat, min_lon, min_lat, max_lon, min_lat), + "-geo:\"Within(ENVELOPE(%s,%s,%s,%s))\"" % (min_lon, max_lon, max_lat, min_lat), + "tile_count_i:[1 TO *]", + time_clause + ] + } + + self._merge_kwargs(additionalparams, **kwargs) + + return self.do_query_all(*(search, None, None, False, None), **additionalparams) + + def do_query(self, *args, **params): + + response = self.do_query_raw(*args, **params) + + return response.results, response.start, response.numFound + + def do_query_raw(self, *args, **params): + + # fl only works when passed as the second argument to solrcon.select + if 'fl' in params.keys(): + fl = params['fl'] + del (params['fl']) + else: + fl = args[1] + + # sort only works when passed as the fourth argument to solrcon.select + if 'sort' in params.keys(): + s = ','.join(params['sort']) + del (params['sort']) + else: + s = args[4] + + # If dataset_s is specified as the search term, + # add the _route_ parameter to limit the search to the correct shard + if 'dataset_s:' in args[0]: + ds = args[0].split(':')[-1] + params['shard_keys'] = ds + '!' + + args = (args[0],) + (fl,) + (args[2:4]) + (s,) + + with SOLR_CON_LOCK: + response = self.solrcon.select(*args, **params) + + return response + + def do_query_all(self, *args, **params): + + results = [] + + response = self.do_query_raw(*args, **params) + results.extend(response.results) + + limit = min(params.get('limit', float('inf')), response.numFound) + + while len(results) < limit: + params['start'] = len(results) + response = self.do_query_raw(*args, **params) + results.extend(response.results) + + assert len(results) == limit + + return results + + def ping(self): + solrAdminPing = 'http://%s/solr/%s/admin/ping' % (self.solrUrl, self.solrCore) + try: + r = requests.get(solrAdminPing, params={'wt': 'json'}) + results = json.loads(r.text) + return results + except: + return None + + @staticmethod + def _merge_kwargs(additionalparams, **kwargs): + # Only Solr-specific kwargs are parsed + # And the special 'limit' + try: + additionalparams['limit'] = kwargs['limit'] + except KeyError: + pass + + try: + additionalparams['_route_'] = kwargs['_route_'] + except KeyError: + pass + + try: + additionalparams['rows'] = kwargs['rows'] + except KeyError: + pass + + try: + additionalparams['start'] = kwargs['start'] + except KeyError: + pass + + try: + kwfq = kwargs['fq'] if isinstance(kwargs['fq'], list) else list(kwargs['fq']) + except KeyError: + kwfq = [] + + try: + additionalparams['fq'].extend(kwfq) + except KeyError: + additionalparams['fq'] = kwfq + + try: + kwfl = kwargs['fl'] if isinstance(kwargs['fl'], list) else [kwargs['fl']] + except KeyError: + kwfl = [] + + try: + additionalparams['fl'].extend(kwfl) + except KeyError: + additionalparams['fl'] = kwfl + + try: + s = kwargs['sort'] if isinstance(kwargs['sort'], list) else [kwargs['sort']] + except KeyError: + s = None + + try: + additionalparams['sort'].extend(s) + except KeyError: + if s is not None: + additionalparams['sort'] = s http://git-wip-us.apache.org/repos/asf/incubator-sdap-nexus/blob/ff98fa34/data-access/nexustiles/dao/__init__.py ---------------------------------------------------------------------- diff --git a/data-access/nexustiles/dao/__init__.py b/data-access/nexustiles/dao/__init__.py new file mode 100644 index 0000000..e69de29
