Author: ahorincar
Date: Tue Jun 24 23:01:55 2014
New Revision: 1605215
URL: http://svn.apache.org/r1605215
Log:
Added pagination, highlighting and faceting to Solr query results
Removed:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/admin.py
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/api.py
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py
Modified:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
Modified:
bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py?rev=1605215&r1=1605214&r2=1605215&view=diff
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
(original)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py
Tue Jun 24 23:01:55 2014
@@ -1,24 +1,52 @@
-import pkg_resources
-
-from solr import Solr
-from trac.ticket.model import Ticket
+from bhsearch import BHSEARCH_CONFIG_SECTION
+from bhsearch.api import ISearchBackend, SCORE, QueryResult
+from bhsearch.query_parser import DefaultQueryParser
+from bhsearch.search_resources.ticket_search import TicketIndexer
from trac.core import Component, implements, TracError
+from trac.config import Option
+from trac.ticket.model import Ticket
from trac.ticket.api import TicketSystem
-from bhsearch.search_resources.ticket_search import TicketIndexer
-from datetime import datetime
from trac.util.datefmt import utc
-from bhsearch.api import ISearchBackend
-from bhsearch.query_parser import DefaultQueryParser
+from datetime import datetime
+from sunburnt import SolrInterface
+from contextlib import contextmanager
+from math import ceil
UNIQUE_ID = "unique_id"
+HIGHLIGHTABLE_FIELDS = {"unique_id" : True,
+ "id" : True,
+ "type" : True,
+ "product" : True,
+ "milestone" : True,
+ "author" : True,
+ "component" : True,
+ "status" : True,
+ "resolution" : True,
+ "keywords" : True,
+ "summary" : True,
+ "content" : True,
+ "changes" : True,
+ "owner" : True,
+ "repository" : True,
+ "revision" : True,
+ "message" : True,
+ "name" : True}
+
class SolrBackend(Component):
implements(ISearchBackend)
+ server_url = Option(
+ BHSEARCH_CONFIG_SECTION,
+ 'solr_server_url',
+ doc="""Url of the server running Solr instance.""",
+ doc_domain='bhsearch')
+
+
def __init__(self):
- resource_filename = pkg_resources.resource_filename
- path = resource_filename(__name__, "schemadoc")
- self.solr_interface = Solr("http://localhost:8983/solr/";).solr_interface #
TODO: Refactor
+ # resource_filename = pkg_resources.resource_filename
+ # path = resource_filename(__name__, "schemadoc") # TODO: Use absolute
path of schema
+ self.solr_interface = SolrInterface(str(self.server_url))
self.field_boosts = DefaultQueryParser(self.env).field_boosts
@@ -27,6 +55,7 @@ class SolrBackend(Component):
doc[UNIQUE_ID] = self._create_unique_id(doc.get("product", ''),
doc["type"],
doc["id"])
+ print doc["type"]
self.solr_interface.add(doc)
self.solr_interface.commit()
@@ -43,6 +72,42 @@ class SolrBackend(Component):
facets = None, pagenum = 1, pagelen = 20, highlight = False,
highlight_fields = None, context = None):
+ final_query_chain = self._create_query_chain(query)
+ solr_query = self.solr_interface.query(final_query_chain)
+ faceted_solr_query = solr_query.facet_by(facets)
+ highlighted_solr_query = faceted_solr_query.highlight(HIGHLIGHTABLE_FIELDS)
+ paginated_solr_query = highlighted_solr_query.paginate(rows=20000)
+
+ results = paginated_solr_query.execute()
+
+ return self._create_query_result(results, fields, pagenum, pagelen)
+
+ def _create_query_result(self, results, fields, pagenum, pagelen):
+ total_num, total_page_count, page_num, offset = \
+ self._prepare_query_result_attributes(results, pagenum,
pagelen)
+
+ query_results = QueryResult()
+ query_results.hits = total_num
+ query_results.total_page_count = total_page_count
+ query_results.page_number = page_num
+ query_results.offset = offset
+
+ docs = []
+ highlighting = []
+
+ for retrieved_record in results:
+ result_doc = self._process_record(fields, retrieved_record)
+ docs.append(result_doc)
+
+ result_highlights = dict(retrieved_record['solr_highlights'])
+
+ highlighting.append(result_highlights)
+ query_results.docs = docs
+ query_results.highlighting = highlighting
+
+ return query_results
+
+ def _create_query_chain(self, query):
tokens = set([token.text for token in query.all_tokens()])
final_query_chain = None
for token in tokens:
@@ -52,8 +117,39 @@ class SolrBackend(Component):
else:
final_query_chain |= token_query_chain
- print final_query_chain
- return None
+ return final_query_chain
+
+ def _process_record(self, fields, retrieved_record):
+ result_doc = dict()
+
+ if fields:
+ for field in fields:
+ if field in retrieved_record:
+ result_doc[field] = retrieved_record[field]
+ else:
+ for key, value in retrieved_record.items():
+ result_doc[key] = value
+
+ for key, value in result_doc.iteritems():
+ result_doc[key] = self._from_solr_format(value)
+
+ return result_doc
+
+ def _from_solr_format(self, value):
+ if isinstance(value, datetime):
+ value = utc.localize(value)
+ return value
+
+ def _prepare_query_result_attributes(self, results, pagenum, pagelen):
+ results_total_num = len(results)
+ total_page_count = int(ceil(results_total_num / pagelen))
+ pagenum = min(total_page_count, pagenum)
+
+ offset = (pagenum - 1) * pagelen
+ if (offset + pagelen) > results_total_num:
+ pagelen = results_total_num - offset
+
+ return results_total_num, total_page_count, pagenum, offset
def is_index_outdated(self):
return False
@@ -61,17 +157,19 @@ class SolrBackend(Component):
def recreate_index(self):
return True
+ @contextmanager
def start_operation(self):
- return None
+ yield
def _search_fields_for_token(self, token):
query_chain = None
for field, boost in self.field_boosts.iteritems():
- field_token_dict = {field: token}
- if query_chain == None:
- query_chain = self.solr_interface.Q(**field_token_dict)**boost
- else:
- query_chain |= self.solr_interface.Q(**field_token_dict)**boost
+ if field != 'query_suggestion_basket' and field != 'relations':
+ field_token_dict = {field: token}
+ if query_chain == None:
+ query_chain = self.solr_interface.Q(**field_token_dict)**boost
+ else:
+ query_chain |= self.solr_interface.Q(**field_token_dict)**boost
return query_chain
@@ -86,7 +184,6 @@ class SolrBackend(Component):
else:
doc[key] = self._to_solr_format(value)
-
def _to_solr_format(self, value):
if isinstance(value, basestring):
value = unicode(value)
Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py
URL:
http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py?rev=1605215&r1=1605214&r2=1605215&view=diff
==============================================================================
--- bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py (original)
+++ bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Tue Jun 24
23:01:55 2014
@@ -9,7 +9,6 @@ PKG_INFO = {'bhsolr': ['schemadoc/*.xml'
ENTRY_POINTS = {
'trac.plugins': [
'bhsolr.api = bhsolr.api',
- 'bhsolr.solr = bhsolr.solr',
'bhsolr.admin = bhsolr.admin',
'bhsolr.solr_backend = bhsolr.solr_backend',
'bhsolr.search_resources.ticket_search =
bhsolr.search_resources.ticket_search',
@@ -25,7 +24,7 @@ setup(
author = "Apache Bloodhound",
license = "Apache License v2",
url = "http://bloodhound.apache.org/";,
- # package_dir = PKG_INFO,
+ requires = ['trac', 'lxml', 'sunburnt', 'httplib2'],
packages = find_packages(),
package_data = PKG_INFO,
include_package_data=True,
