Maximiliano Bertacchini has proposed merging
lp:~maxiberta/launchpad/bing-search into lp:launchpad.
Commit message:
Add basic Bing Custom Search support.
Requested reviews:
Launchpad code reviewers (launchpad-reviewers)
For more details, see:
https://code.launchpad.net/~maxiberta/launchpad/bing-search/+merge/341549
Add basic Bing Custom Search support.
This should be pretty unobtrusive, while adding a basic implementation around
Bing Custom Search. Needs more testing.
--
Your team Launchpad code reviewers is requested to review the proposed merge of
lp:~maxiberta/launchpad/bing-search into lp:launchpad.
=== modified file 'Makefile'
--- Makefile 2018-02-01 20:56:23 +0000
+++ Makefile 2018-03-16 21:18:39 +0000
@@ -61,6 +61,7 @@
# NB: It's important PIP_BIN only mentions things genuinely produced by pip.
PIP_BIN = \
$(PY) \
+ bin/bingtestservice \
bin/build-twisted-plugin-cache \
bin/combine-css \
bin/googletestservice \
@@ -284,7 +285,7 @@
bin/test -f $(TESTFLAGS) $(TESTOPTS)
run: build inplace stop
- bin/run -r librarian,google-webservice,memcached,rabbitmq,txlongpoll \
+ bin/run -r librarian,bing-webservice,memcached,rabbitmq,txlongpoll \
-i $(LPCONFIG)
run-testapp: LPCONFIG=testrunner-appserver
@@ -297,12 +298,12 @@
start-gdb: build inplace stop support_files run.gdb
nohup gdb -x run.gdb --args bin/run -i $(LPCONFIG) \
- -r librarian,google-webservice
+ -r librarian,bing-webservice
> ${LPCONFIG}-nohup.out 2>&1 &
run_all: build inplace stop
bin/run \
- -r librarian,sftp,forker,mailman,codebrowse,google-webservice,\
+ -r librarian,sftp,forker,mailman,codebrowse,bing-webservice,\
memcached,rabbitmq,txlongpoll -i $(LPCONFIG)
run_codebrowse: compile
=== modified file 'configs/development/launchpad-lazr.conf'
--- configs/development/launchpad-lazr.conf 2018-02-02 15:29:38 +0000
+++ configs/development/launchpad-lazr.conf 2018-03-16 21:18:39 +0000
@@ -86,6 +86,15 @@
[google_test_service]
launch: True
+[bing]
+# Development and the testrunner should use the stub service be default.
+site: http://launchpad.dev:8093/bingcustomsearch/v7.0/search
+subscription_key: abcdef01234567890abcdef012345678
+custom_config_id: 1234567890
+
+[bing_test_service]
+launch: True
+
[gpghandler]
host: keyserver.launchpad.dev
public_host: keyserver.launchpad.dev
=== modified file 'configs/testrunner-appserver/launchpad-lazr.conf'
--- configs/testrunner-appserver/launchpad-lazr.conf 2016-10-11 15:28:25 +0000
+++ configs/testrunner-appserver/launchpad-lazr.conf 2018-03-16 21:18:39 +0000
@@ -14,6 +14,9 @@
[google_test_service]
launch: False
+[bing_test_service]
+launch: False
+
[launchpad]
openid_provider_root: http://testopenid.dev:8085/
=== modified file 'configs/testrunner/launchpad-lazr.conf'
--- configs/testrunner/launchpad-lazr.conf 2018-01-26 22:18:38 +0000
+++ configs/testrunner/launchpad-lazr.conf 2018-03-16 21:18:39 +0000
@@ -91,6 +91,9 @@
[google]
site: http://launchpad.dev:8092/cse
+[bing]
+site: http://launchpad.dev:8093/bingcustomsearch/v7.0/search
+
[gpghandler]
upload_keys: True
host: localhost
=== modified file 'lib/lp/app/browser/root.py'
--- lib/lp/app/browser/root.py 2018-03-16 14:50:01 +0000
+++ lib/lp/app/browser/root.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Browser code for the Launchpad root page."""
@@ -45,6 +45,13 @@
GoogleResponseError,
ISearchService,
)
+from lp.services.memcache.interfaces import IMemcacheClient
+from lp.services.propertycache import cachedproperty
+from lp.services.sitesearch.interfaces import (
+ BingResponseError,
+ GoogleResponseError,
+ ISearchService,
+ )
from lp.services.statistics.interfaces.statistic import ILaunchpadStatisticSet
from lp.services.timeout import urlfetch
from lp.services.webapp import LaunchpadView
@@ -510,24 +517,24 @@
def searchPages(self, query_terms, start=0):
"""Return the up to 20 pages that match the query_terms, or None.
- :param query_terms: The unescaped terms to query Google.
+ :param query_terms: The unescaped terms to query for.
:param start: The index of the page that starts the set of pages.
- :return: A GooglBatchNavigator or None.
+ :return: A SearchResultsBatchNavigator or None.
"""
if query_terms in [None, '']:
return None
- google_search = getUtility(ISearchService)
+ site_search = getUtility(ISearchService)
try:
- page_matches = google_search.search(
+ page_matches = site_search.search(
terms=query_terms, start=start)
- except GoogleResponseError:
- # There was a connectivity or Google service issue that means
+ except (BingResponseError, GoogleResponseError):
+ # There was a connectivity or the search service issue that means
# there is no data available at this moment.
self.has_page_service = False
return None
if len(page_matches) == 0:
return None
- navigator = GoogleBatchNavigator(
+ navigator = SearchResultsBatchNavigator(
page_matches, self.request, start=start)
navigator.setHeadings(*self.batch_heading)
return navigator
@@ -589,7 +596,7 @@
return self.start + len(self.list._window)
-class GoogleBatchNavigator(BatchNavigator):
+class SearchResultsBatchNavigator(BatchNavigator):
"""A batch navigator with a fixed size of 20 items per batch."""
_batch_factory = WindowedListBatch
@@ -614,7 +621,7 @@
:param callback: Not used.
"""
results = WindowedList(results, start, results.total)
- super(GoogleBatchNavigator, self).__init__(results, request,
+ super(SearchResultsBatchNavigator, self).__init__(results, request,
start=start, size=size, callback=callback,
transient_parameters=transient_parameters,
force_start=force_start, range_factory=range_factory)
=== modified file 'lib/lp/app/browser/tests/launchpad-search-pages.txt'
--- lib/lp/app/browser/tests/launchpad-search-pages.txt 2018-03-16 14:02:16 +0000
+++ lib/lp/app/browser/tests/launchpad-search-pages.txt 2018-03-16 21:18:39 +0000
@@ -416,7 +416,7 @@
>>> search_view.has_matches
True
>>> search_view.pages
- <...GoogleBatchNavigator ...>
+ <...SiteSearchBatchNavigator ...>
The GoogleSearchService may not be available due to connectivity problems.
The view's has_page_service attribute reports when the search was performed
@@ -444,7 +444,7 @@
>>> search_view.batch_heading
(u'other page matching "launchpad"', u'other pages matching "launchpad"')
-The GoogleBatchNavigator behaves like most BatchNavigators, except that
+The SiteSearchBatchNavigator behaves like most BatchNavigators, except that
its batch size is always 20. The size restriction conforms to Google's
maximum number of results that can be returned per request.
@@ -625,7 +625,7 @@
</div>
</form>
-WindowedList and GoogleBatchNavigator
+WindowedList and SiteSearchBatchNavigator
-------------------------------------
The LaunchpadSearchView uses two helper classes to work with
@@ -635,7 +635,7 @@
or fewer PageMatches of what could be thousands of matches. Google
requires client's to make repeats request to step though the batches of
matches. The Windowed list is a list that contains only a subset of its
-reported size. It is used to make batches in the GoogleBatchNavigator.
+reported size. It is used to make batches in the SiteSearchBatchNavigator.
For example, the last batch of the 'bug' search contained 5 of the 25
matching pages. The WindowList claims to be 25 items in length, but
@@ -657,14 +657,14 @@
>>> results[18, 22]
[None, None, <...PageMatch ...>, <...PageMatch ...>]
-The GoogleBatchNavigator restricts the batch size to 20. the 'batch'
+The SiteSearchBatchNavigator restricts the batch size to 20. the 'batch'
parameter that comes from the URL is ignored. For example, setting
the 'batch' parameter to 100 has no affect upon the Google search
or on the navigator object.
- >>> from lp.app.browser.root import GoogleBatchNavigator
+ >>> from lp.app.browser.root import SiteSearchBatchNavigator
- >>> GoogleBatchNavigator.batch_variable_name
+ >>> SiteSearchBatchNavigator.batch_variable_name
'batch'
>>> search_view = getSearchView(
@@ -687,7 +687,7 @@
>>> page_matches._matches = matches
>>> page_matches.start = 0
>>> page_matches.total = 100
- >>> navigator = GoogleBatchNavigator(
+ >>> navigator = SiteSearchBatchNavigator(
... page_matches, search_view.request, page_matches.start, size=100)
>>> navigator.currentBatch().size
20
@@ -705,7 +705,7 @@
>>> matches = list(range(0, 3))
>>> page_matches._matches = matches
- >>> navigator = GoogleBatchNavigator(
+ >>> navigator = SiteSearchBatchNavigator(
... page_matches, search_view.request, page_matches.start, size=100)
>>> batch = navigator.currentBatch()
>>> batch.size
=== modified file 'lib/lp/app/browser/tests/test_views.py'
--- lib/lp/app/browser/tests/test_views.py 2011-12-28 17:03:06 +0000
+++ lib/lp/app/browser/tests/test_views.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""
@@ -11,7 +11,7 @@
from lp.testing.layers import (
DatabaseFunctionalLayer,
- GoogleLaunchpadFunctionalLayer,
+ BingLaunchpadFunctionalLayer,
)
from lp.testing.systemdocs import (
LayeredDocFileSuite,
@@ -26,7 +26,7 @@
# that require something special like the librarian or mailman must run
# on a layer that sets those services up.
special_test_layer = {
- 'launchpad-search-pages.txt': GoogleLaunchpadFunctionalLayer,
+ 'launchpad-search-pages.txt': BingLaunchpadFunctionalLayer,
}
=== modified file 'lib/lp/scripts/runlaunchpad.py'
--- lib/lp/scripts/runlaunchpad.py 2018-03-16 14:50:01 +0000
+++ lib/lp/scripts/runlaunchpad.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2017 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
__metaclass__ = type
@@ -26,7 +26,10 @@
pidfile_path,
)
from lp.services.rabbit.server import RabbitServer
-from lp.services.sitesearch import googletestservice
+from lp.services.sitesearch import (
+ bingtestservice,
+ googletestservice,
+ )
from lp.services.txlongpoll.server import TxLongPollServer
@@ -153,6 +156,16 @@
self.addCleanup(stop_process, googletestservice.start_as_process())
+class BingWebService(Service):
+
+ @property
+ def should_launch(self):
+ return config.bing_test_service.launch
+
+ def launch(self):
+ self.addCleanup(stop_process, bingtestservice.start_as_process())
+
+
class MemcachedService(Service):
"""A local memcached service for developer environments."""
@@ -280,6 +293,7 @@
'sftp': TacFile('sftp', 'daemons/sftp.tac', 'codehosting'),
'forker': ForkingSessionService(),
'mailman': MailmanService(),
+ 'bing-webservice': BingWebService(),
'codebrowse': CodebrowseService(),
'google-webservice': GoogleWebService(),
'memcached': MemcachedService(),
=== modified file 'lib/lp/services/config/schema-lazr.conf'
--- lib/lp/services/config/schema-lazr.conf 2018-03-16 14:02:16 +0000
+++ lib/lp/services/config/schema-lazr.conf 2018-03-16 21:18:39 +0000
@@ -794,6 +794,42 @@
# Example: help.launchpad.net login.launchapd.net
url_rewrite_exceptions: help.launchpad.net
+[bing_test_service]
+# Run a web service stub that simulates the Bing search service.
+
+# Where are our canned JSON responses stored?
+canned_response_directory: lib/lp/services/sitesearch/tests/data/
+
+# Which file maps service URLs to the JSON that the server returns?
+mapfile: lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt
+
+# Where should the service log files live?
+log: logs/bing-stub.log
+
+# Do we actually want to run the service?
+launch: False
+
+[bing]
+# site is the host and path that search requests are made to.
+# eg. https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search
+# datatype: string, a url to a host
+site: https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search
+
+# subscription_key is the Cognitive Services subscription key for
+# Bing Custom Search API.
+# datatype: string
+subscription_key:
+
+# custom_config_id is the id that identifies the custom search instance.
+# datatype: string
+custom_config_id:
+
+# url_rewrite_exceptions is a list of launchpad.net domains that must
+# not be rewritten.
+# datatype: string of space separated domains
+# Example: help.launchpad.net login.launchapd.net
+url_rewrite_exceptions: help.launchpad.net
+
[gpghandler]
# Should we allow uploading keys to the keyserver?
# datatype: boolean
=== modified file 'lib/lp/services/sitesearch/__init__.py'
--- lib/lp/services/sitesearch/__init__.py 2018-03-16 14:02:16 +0000
+++ lib/lp/services/sitesearch/__init__.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interfaces for searching and working with results."""
@@ -6,11 +6,13 @@
__metaclass__ = type
__all__ = [
+ 'BingSearchService',
'GoogleSearchService',
'PageMatch',
'PageMatches',
]
+import json
import urllib
from urlparse import (
parse_qsl,
@@ -25,6 +27,7 @@
from lp.services.config import config
from lp.services.sitesearch.interfaces import (
+ BingResponseError,
GoogleResponseError,
GoogleWrongGSPVersion,
ISearchResult,
@@ -337,3 +340,161 @@
raise GoogleWrongGSPVersion(
"Could not get any PageMatches from the GSP XML response.")
return PageMatches(page_matches, start, total)
+
+
+@implementer(ISearchService)
+class BingSearchService:
+ """See `ISearchService`.
+
+ A search service that search Bing for launchpad.net pages.
+ """
+
+ _default_values = {
+ 'customConfig': None,
+ 'mkt': 'en-US', # FIXME: set language based on the current request?
+ 'count': 20,
+ 'offset': 0,
+ 'q': None,
+ }
+
+ @property
+ def subscription_key(self):
+ """The subscription key issued by Bing Custom Search."""
+ return config.bing.subscription_key
+
+ @property
+ def custom_config_id(self):
+ """The custom search instance as configured in Bing Custom Search."""
+ return config.bing.custom_config_id
+
+ @property
+ def site(self):
+ """The URL to the Bing Custom Search service.
+
+ The URL is probably
+ https://api.cognitive.microsoft.com/bingcustomsearch/v7.0/search.
+ """
+ return config.bing.site
+
+ def search(self, terms, start=0):
+ """See `ISearchService`.
+
+ The `subscription_key` and `custom_config_id` are used in the
+ search request. Search returns 20 or fewer results for each query.
+ For terms that match more than 20 results, the start param can be
+ used over multiple queries to get successive sets of results.
+
+ :return: `ISearchResults` (PageMatches).
+ :raise: `BingResponseError` if the json response is incomplete or
+ cannot be parsed.
+ """
+ search_url = self.create_search_url(terms, start=start)
+ search_headers = self.create_search_headers()
+ request = get_current_browser_request()
+ timeline = get_request_timeline(request)
+ action = timeline.start("bing-search-api", search_url)
+ try:
+ response = urlfetch(search_url, headers=search_headers)
+ except (TimeoutError, requests.RequestException) as error:
+ raise BingResponseError("The response errored: %s" % str(error))
+ finally:
+ action.finish()
+ page_matches = self._parse_bing_response(response.content, start)
+ return page_matches
+
+ def _checkParameter(self, name, value, is_int=False):
+ """Check that a parameter value is not None or an empty string."""
+ if value in (None, ''):
+ raise AssertionError("Missing value for parameter '%s'." % name)
+ if is_int:
+ try:
+ int(value)
+ except ValueError:
+ raise AssertionError(
+ "Value for parameter '%s' is not an int." % name)
+
+ def create_search_url(self, terms, start=0):
+ """Return a Bing Custom Search search url."""
+ self._checkParameter('q', terms)
+ self._checkParameter('offset', start, is_int=True)
+ self._checkParameter('customconfig', self.custom_config_id)
+ safe_terms = urllib.quote_plus(terms.encode('utf8'))
+ search_params = dict(self._default_values)
+ search_params['q'] = safe_terms
+ search_params['offset'] = start
+ search_params['customConfig'] = self.custom_config_id
+ search_param_list = [
+ '%s=%s' % (name, value)
+ for name, value in sorted(search_params.items())]
+ query_string = '&'.join(search_param_list)
+ return self.site + '?' + query_string
+
+ def create_search_headers(self):
+ """Return a dict with Bing Custom Search compatible request headers."""
+ self._checkParameter('subscription_key', self.subscription_key)
+ return {
+ 'Ocp-Apim-Subscription-Key': self.subscription_key,
+ }
+
+ def _parse_bing_response(self, bing_json, start=0):
+ """Return a `PageMatches` object.
+
+ :param bing_json: A string containing Bing Custom Search API v7 JSON.
+ :return: `ISearchResults` (PageMatches).
+ :raise: `BingResponseError` if the json response is incomplete or
+ cannot be parsed.
+ """
+ try:
+ bing_doc = json.loads(bing_json)
+ except (TypeError, ValueError):
+ raise BingResponseError("The response was incomplete, no JSON.")
+
+ try:
+ response_type = bing_doc['_type']
+ except (AttributeError, KeyError, ValueError):
+ raise BingResponseError(
+ "Could not get the '_type' from the Bing JSON response.")
+
+ if response_type == 'ErrorResponse':
+ try:
+ errors = [error['message'] for error in bing_doc['errors']]
+ raise BingResponseError(
+ "Error response from Bing: %s" % '; '.join(errors))
+ except (AttributeError, KeyError, TypeError, ValueError):
+ raise BingResponseError(
+ "Unable to parse the Bing JSON error response.")
+ elif response_type != 'SearchResponse':
+ raise BingResponseError(
+ "Unknown Bing JSON response type: '%s'." % response_type)
+
+ page_matches = []
+ total = 0
+ try:
+ results = bing_doc['webPages']['value']
+ except (AttributeError, KeyError, ValueError):
+ # Bing did not match any pages. Return an empty PageMatches.
+ return PageMatches(page_matches, start, total)
+
+ try:
+ total = int(bing_doc['webPages']['totalEstimatedMatches'])
+ except (AttributeError, KeyError, ValueError):
+ # The datatype is not what PageMatches requires.
+ raise GoogleWrongGSPVersion(
+ "Could not get the total from the Bing JSON response.")
+ if total < 0:
+ # See bug 683115.
+ total = 0
+ for result in results:
+ url = result.get('url')
+ title = result.get('name')
+ summary = result.get('snippet')
+ if None in (url, title, summary):
+ # There is not enough data to create a PageMatch object.
+ # This can be caused by an empty title or summary which
+ # has been observed for pages that are from vhosts that
+ # should not be indexed.
+ continue
+ summary = summary.replace('<br>', '')
+ page_matches.append(PageMatch(title, url, summary))
+
+ return PageMatches(page_matches, start, total)
=== added file 'lib/lp/services/sitesearch/bingtestservice.py'
--- lib/lp/services/sitesearch/bingtestservice.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/bingtestservice.py 2018-03-16 21:18:39 +0000
@@ -0,0 +1,271 @@
+#!/usr/bin/python
+#
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+This script runs a simple HTTP server. The server returns JSON files
+when given certain user-configurable URLs.
+"""
+
+
+from BaseHTTPServer import (
+ BaseHTTPRequestHandler,
+ HTTPServer,
+ )
+import errno
+import logging
+import os
+import signal
+import socket
+import subprocess
+import time
+
+from lp.services.config import config
+from lp.services.osutils import ensure_directory_exists
+from lp.services.pidfile import (
+ get_pid,
+ make_pidfile,
+ pidfile_path,
+ )
+from lp.services.webapp.url import urlsplit
+
+# Set up basic logging.
+log = logging.getLogger(__name__)
+
+# The default service name, used by the Launchpad service framework.
+service_name = 'bing-webservice'
+
+
+class BingRequestHandler(BaseHTTPRequestHandler):
+ """Return an JSON file depending on the requested URL."""
+
+ default_content_type = 'application/json; charset=utf-8'
+
+ def do_GET(self):
+ """See BaseHTTPRequestHandler in the Python Standard Library."""
+ urlmap = url_to_json_map()
+ if self.path in urlmap:
+ self.return_file(urlmap[self.path])
+ else:
+ # Return our default route.
+ self.return_file(urlmap['*'])
+
+ def return_file(self, filename):
+ """Return a HTTP response with 'filename' for content.
+
+ :param filename: The file name to find in the canned-data
+ storage location.
+ """
+ self.send_response(200)
+ self.send_header('Content-Type', self.default_content_type)
+ self.end_headers()
+
+ content_dir = config.bing_test_service.canned_response_directory
+ filepath = os.path.join(content_dir, filename)
+ content_body = file(filepath).read()
+ self.wfile.write(content_body)
+
+ def log_message(self, format, *args):
+ """See `BaseHTTPRequestHandler.log_message()`."""
+ # Substitute the base class's logger with the Python Standard
+ # Library logger.
+ message = ("%s - - [%s] %s" %
+ (self.address_string(),
+ self.log_date_time_string(),
+ format % args))
+ log.info(message)
+
+
+def url_to_json_map():
+ """Return our URL-to-JSON mapping as a dictionary."""
+ mapfile = config.bing_test_service.mapfile
+ mapping = {}
+ for line in file(mapfile):
+ if line.startswith('#') or len(line.strip()) == 0:
+ # Skip comments and blank lines.
+ continue
+ url, fname = line.split()
+ mapping[url.strip()] = fname.strip()
+
+ return mapping
+
+
+def get_service_endpoint():
+ """Return the host and port that the service is running on."""
+ return hostpair(config.bing.site)
+
+
+def service_is_available(timeout=2.0):
+ """Return True if the service is up and running.
+
+ :param timeout: BLOCK execution for at most 'timeout' seconds
+ before returning False.
+ """
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(timeout) # Block for 'timeout' seconds.
+ host, port = get_service_endpoint()
+ try:
+ try:
+ sock.connect((host, port))
+ except socket.error:
+ return False
+ else:
+ return True
+ finally:
+ sock.close() # Clean up.
+
+
+def wait_for_service(timeout=15.0):
+ """Poll the service and BLOCK until we can connect to it.
+
+ :param timeout: The socket should timeout after this many seconds.
+ Refer to the socket module documentation in the Standard Library
+ for possible timeout values.
+ """
+ host, port = get_service_endpoint()
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(timeout) # Block for at most X seconds.
+
+ start = time.time() # Record when we started polling.
+ try:
+ while True:
+ try:
+ sock.connect((host, port))
+ except socket.error as err:
+ if err.args[0] in [errno.ECONNREFUSED, errno.ECONNABORTED]:
+ elapsed = (time.time() - start)
+ if elapsed > timeout:
+ raise RuntimeError("Socket poll time exceeded.")
+ else:
+ raise
+ else:
+ break
+ time.sleep(0.1)
+ finally:
+ sock.close() # Clean up.
+
+
+def wait_for_service_shutdown(seconds_to_wait=10.0):
+ """Poll the service until it shuts down.
+
+ Raises a RuntimeError if the service doesn't shut down within the allotted
+ time, under normal operation. It may also raise various socket errors if
+ there are issues connecting to the service (host lookup, etc.)
+
+ :param seconds_to_wait: The number of seconds to wait for the socket to
+ open up.
+ """
+ host, port = get_service_endpoint()
+
+ start = time.time() # Record when we started polling.
+ try:
+ while True:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(5.0) # Block for at most X seconds.
+ try:
+ sock.connect((host, port))
+ sock.close()
+ except socket.error as err:
+ if err.args[0] == errno.ECONNREFUSED:
+ # Success! The socket is closed.
+ return
+ else:
+ raise
+ else:
+ elapsed = (time.time() - start)
+ if elapsed > seconds_to_wait:
+ raise RuntimeError(
+ "The service did not shut down in the allotted time.")
+ time.sleep(0.1)
+ finally:
+ sock.close() # Clean up.
+
+
+def hostpair(url):
+ """Parse the host and port number out of a URL string."""
+ parts = urlsplit(url)
+ host, port = parts[1].split(':')
+ port = int(port)
+ return (host, port)
+
+
+def start_as_process():
+ """Run this file as a stand-alone Python script.
+
+ Returns a subprocess.Popen object. (See the `subprocess` module in
+ the Python Standard Library for details.)
+ """
+ script = os.path.join(
+ os.path.dirname(__file__),
+ os.pardir, os.pardir, os.pardir, os.pardir, 'bin',
+ 'bingtestservice')
+ # Make sure we aren't using the parent stdin and stdout to avoid spam
+ # and have fewer things that can go wrong shutting down the process.
+ proc = subprocess.Popen(
+ script, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ proc.stdin.close()
+ return proc
+
+
+def kill_running_process():
+ """Find and kill any running web service processes."""
+ global service_name
+ try:
+ pid = get_pid(service_name)
+ except IOError:
+ # We could not find an existing pidfile.
+ return
+ except ValueError:
+ # The file contained a mangled and invalid PID number, so we should
+ # clean the file up.
+ safe_unlink(pidfile_path(service_name))
+ else:
+ if pid is not None:
+ try:
+ os.kill(pid, signal.SIGTERM)
+ # We need to use a busy-wait to find out when the socket
+ # becomes available. Failing to do so causes a race condition
+ # between freeing the socket in the killed process, and
+ # opening it in the current one.
+ wait_for_service_shutdown()
+ except os.error as err:
+ if err.errno == errno.ESRCH:
+ # Whoops, we got a 'No such process' error. The PID file
+ # is probably stale, so we'll remove it to prevent trash
+ # from lying around in the test environment.
+ # See bug #237086.
+ safe_unlink(pidfile_path(service_name))
+ else:
+ raise
+
+
+def safe_unlink(filepath):
+ """Unlink a file, but don't raise an error if the file is missing."""
+ try:
+ os.unlink(filepath)
+ except os.error as err:
+ if err.errno != errno.ENOENT:
+ raise
+
+
+def main():
+ """Run the HTTP server."""
+ # Redirect our service output to a log file.
+ global log
+ ensure_directory_exists(os.path.dirname(config.bing_test_service.log))
+ filelog = logging.FileHandler(config.bing_test_service.log)
+ log.addHandler(filelog)
+ log.setLevel(logging.DEBUG)
+
+ # To support service shutdown we need to create a PID file that is
+ # understood by the Launchpad services framework.
+ global service_name
+ make_pidfile(service_name)
+
+ host, port = get_service_endpoint()
+ server = HTTPServer((host, port), BingRequestHandler)
+
+ log.info("Starting HTTP Bing webservice server on port %s", port)
+ server.serve_forever()
=== modified file 'lib/lp/services/sitesearch/interfaces.py'
--- lib/lp/services/sitesearch/interfaces.py 2013-01-07 02:40:55 +0000
+++ lib/lp/services/sitesearch/interfaces.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Interfaces for searching and working with results."""
@@ -6,6 +6,7 @@
__metaclass__ = type
__all__ = [
+ 'BingResponseError',
'ISearchResult',
'ISearchResults',
'ISearchService',
@@ -75,7 +76,11 @@
class GoogleResponseError(SyntaxError):
- """Raised when Google's response is not contain valid XML."""
+ """Raised when Google's response does not contain valid XML."""
+
+
+class BingResponseError(ValueError):
+ """Raised when Bing's response does not contain valid JSON."""
class ISearchService(Interface):
=== added file 'lib/lp/services/sitesearch/tests/bingserviceharness.py'
--- lib/lp/services/sitesearch/tests/bingserviceharness.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/bingserviceharness.py 2018-03-16 21:18:39 +0000
@@ -0,0 +1,107 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+Fixtures for running the Bing test webservice.
+"""
+
+__metaclass__ = type
+
+__all__ = ['BingServiceTestSetup']
+
+
+import errno
+import os
+import signal
+
+from lp.services.sitesearch import bingtestservice
+
+
+class BingServiceTestSetup:
+ """Set up the Bing web service stub for use in functional tests.
+ """
+
+ # XXX gary 2008-12-06 bug=305858: Spurious test failures discovered on
+ # buildbot, builds 40 and 43. The locations of the failures are marked
+ # below with " # SPURIOUS FAILURE". To reinstate, add the text below back
+ # to the docstring above. Note that the test that uses this setup,
+ # bing-service-stub.txt, is also disabled. See test_doc.py.
+ """
+ >>> from lp.services.sitesearch.bingtestservice import (
+ ... service_is_available)
+ >>> from lp.services.config import config
+
+ >>> assert not service_is_available() # Sanity check. # SPURIOUS FAILURE
+
+ >>> BingServiceTestSetup().setUp()
+
+ After setUp is called, a Bing test service instance is running.
+
+ >>> assert service_is_available()
+ >>> assert BingServiceTestSetup.service is not None
+
+ After tearDown is called, the service is shut down.
+
+ >>> BingServiceTestSetup().tearDown()
+
+ >>> assert not service_is_available()
+ >>> assert BingServiceTestSetup.service is None
+
+ The fixture can be started and stopped multiple time in succession:
+
+ >>> BingServiceTestSetup().setUp()
+ >>> assert service_is_available()
+
+ Having a service instance already running doesn't prevent a new
+ service from starting. The old instance is killed off and replaced
+ by the new one.
+
+ >>> old_pid = BingServiceTestSetup.service.pid
+ >>> BingServiceTestSetup().setUp() # SPURIOUS FAILURE
+ >>> BingServiceTestSetup.service.pid != old_pid
+ True
+
+ Tidy up.
+
+ >>> BingServiceTestSetup().tearDown()
+ >>> assert not service_is_available()
+
+ """
+
+ service = None # A reference to our running service.
+
+ def setUp(self):
+ self.startService()
+
+ def tearDown(self):
+ self.stopService()
+
+ @classmethod
+ def startService(cls):
+ """Start the webservice."""
+ bingtestservice.kill_running_process()
+ cls.service = bingtestservice.start_as_process()
+ assert cls.service, "The Search service process did not start."
+ try:
+ bingtestservice.wait_for_service()
+ except RuntimeError:
+ # The service didn't start itself soon enough. We must
+ # make sure to kill any errant processes that may be
+ # hanging around.
+ cls.stopService()
+ raise
+
+ @classmethod
+ def stopService(cls):
+ """Shut down the webservice instance."""
+ if cls.service:
+ try:
+ os.kill(cls.service.pid, signal.SIGTERM)
+ except OSError as error:
+ if error.errno != errno.ESRCH:
+ raise
+ # The process with the given pid doesn't exist, so there's
+ # nothing to kill or wait for.
+ else:
+ cls.service.wait()
+ cls.service = None
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-1.json 2018-03-16 21:18:39 +0000
@@ -0,0 +1,331 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=41744312F5E645D58DE6733982EDC72A&CID=222E331334AA69A5318C38A7350C683B&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=41744312F5E645D58DE6733982EDC72A&CID=222E331334AA69A5318C38A7350C683B&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "bug"
+ },
+ "webPages": {
+ "webSearchUrl": "https://www.bing.com/search?q=bug";,
+ "webSearchUrlPingSuffix": "DevEx,5530.1",
+ "totalEstimatedMatches": 87000,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Bugs - Launchpad Help",
+ "url": "https://help.launchpad.net/Bugs";,
+ "urlPingSuffix": "DevEx,5103.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/Bugs";,
+ "snippet": "Launchpad Help > Bugs . Use Launchpad's bug tracker for your project. Bug heat: a computed estimate of a bug's significance. Learn about automatic bug expiry",
+ "deepLinks": [
+ {
+ "name": "Bugs/EmailInterface",
+ "url": "https://help.launchpad.net/Bugs/EmailInterface";,
+ "urlPingSuffix": "DevEx,5093.1",
+ "snippet": "Overview. Launchpad's bug tracker sends you email about the bugs you're interested in. If you see something that requires your attention - for example, you want to ..."
+ },
+ {
+ "name": "Bugs/PluginAPISpec",
+ "url": "https://help.launchpad.net/Bugs/PluginAPISpec";,
+ "urlPingSuffix": "DevEx,5094.1",
+ "snippet": "Overview. We want Launchpad to share bug reports, comments, statuses and other information with as many bug trackers as possible. We've already produced plugins that ..."
+ },
+ {
+ "name": "Bugs/Subscriptions",
+ "url": "https://help.launchpad.net/Bugs/Subscriptions";,
+ "urlPingSuffix": "DevEx,5095.1",
+ "snippet": "Overview. Launchpad uses notification emails and Atom feeds to help you stay on top of the bugs that interest you. Bug mail. There are three ways to get bug ..."
+ },
+ {
+ "name": "Bugs/YourProject",
+ "url": "https://help.launchpad.net/Bugs/YourProject";,
+ "urlPingSuffix": "DevEx,5096.1",
+ "snippet": "Overview. Launchpad's bug tracker is unique: it can track how one bug affects different communities. When you share free software, you share its bugs."
+ },
+ {
+ "name": "Bugs/Expiry",
+ "url": "https://help.launchpad.net/Bugs/Expiry";,
+ "urlPingSuffix": "DevEx,5097.1",
+ "snippet": "This gives you three benefits: you can view a report of all bugs that are due to expiry and so deal with any that need your attention ; once the bug's expired ..."
+ }
+ ],
+ "dateLastCrawled": "2018-02-20T23:45:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";,
+ "name": "Of Bugs and Statuses - Launchpad Blog",
+ "url": "https://blog.launchpad.net/general/of-bugs-and-statuses";,
+ "urlPingSuffix": "DevEx,5149.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://blog.launchpad.net/general/of-bugs-and-statuses";,
+ "snippet": "This past week’s Bug Janitor thread has made it clear that we need better descriptions of what our bug statuses actually mean. While it’s true that many project ...",
+ "dateLastCrawled": "2018-03-09T07:40:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "name": "Mahara 1.8.0",
+ "url": "https://launchpad.net/mahara/+milestone/1.8.0";,
+ "urlPingSuffix": "DevEx,5178.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mahara/+milestone/1.8.0";,
+ "snippet": "Mahara 1.8.0 Release Notes. This is a stable release of Mahara 1.8. Stable releases are fit for general use. If you find a bug, please report it to the tracker:",
+ "dateLastCrawled": "2018-03-04T10:53:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";,
+ "name": "Mighty Box in Launchpad",
+ "url": "https://launchpad.net/mb";,
+ "urlPingSuffix": "DevEx,5193.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mb";,
+ "snippet": "All bugs Latest bugs reported. Bug #766411: The settings dialog box is not available on Windows 7 Reported on 2011-04-19 Bug #766410: App crashes when navigating file ...",
+ "dateLastCrawled": "2018-03-10T07:57:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";,
+ "name": "Bug tracking - Launchpad Bugs",
+ "url": "https://launchpad.net/bugs";,
+ "urlPingSuffix": "DevEx,5208.1",
+ "about": [
+ {
+ "name": "Launchpad"
+ }
+ ],
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/bugs";,
+ "snippet": "Statistics. 1755185 bugs reported across 12480 projects including 122769 links to 2796 bug trackers; 148160 bugs are shared across multiple projects",
+ "dateLastCrawled": "2018-03-14T01:18:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";,
+ "name": "LaunchpadBugTags - Launchpad Development",
+ "url": "https://dev.launchpad.net/LaunchpadBugTags";,
+ "urlPingSuffix": "DevEx,5223.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://dev.launchpad.net/LaunchpadBugTags";,
+ "snippet": "Tagging bugs about Launchpad. Launchpad's Bug Tracker allows you to create ad-hoc groups of bugs with tagging. In the Launchpad team, we have a list of agreed tags ...",
+ "dateLastCrawled": "2018-02-13T19:01:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";,
+ "name": "Evergreen Bug Tracker - Launchpad",
+ "url": "https://launchpad.net/evergreen";,
+ "urlPingSuffix": "DevEx,5238.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/evergreen";,
+ "snippet": "Evergreen is a highly-scalable library system that helps library patrons find library materials, and helps libraries manage, catalog, and circulate those materials ...",
+ "dateLastCrawled": "2018-03-02T10:11:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";,
+ "name": "Ubuntu BugSquad in Launchpad",
+ "url": "https://launchpad.net/~bugsquad";,
+ "urlPingSuffix": "DevEx,5268.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~bugsquad";,
+ "snippet": "http://wiki.ubuntu.com/HelpingWithBugs http://wiki.ubuntu.com/BugSquad The BugSquad is Ubuntu's team for helping out with improving bug reports.",
+ "dateLastCrawled": "2018-02-23T13:05:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";,
+ "name": "Bugs : Launchpad Suite",
+ "url": "https://launchpad.net/launchpad-project/+bugs?field.tag=bugwatch";,
+ "urlPingSuffix": "DevEx,5281.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/launchpad-project/+bugs?field.tag=bugwatch";,
+ "snippet": "Launchpad is the software running this Web site. It includes a project registry, code branch registry and mirroring service, bug tracker, specification tracker ...",
+ "dateLastCrawled": "2017-10-18T02:06:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";,
+ "name": "ufw project files : ufw",
+ "url": "https://launchpad.net/ufw/+download";,
+ "urlPingSuffix": "DevEx,5294.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/ufw/+download";,
+ "snippet": "ufw 0.35 adds support for comments, moves the ufw managed user rules to /etc, improved setup options and several bug fixes.",
+ "dateLastCrawled": "2018-02-19T14:58:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";,
+ "name": "VersionThreeDotO/Bugs/StoryCards - Launchpad Development",
+ "url": "https://dev.launchpad.net/VersionThreeDotO/Bugs/StoryCards";,
+ "urlPingSuffix": "DevEx,5309.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://dev.launchpad.net/VersionThreeDotO/Bugs/StoryCards";,
+ "snippet": "Contents. HWDB. Get number of affected users by a bug tied to a device; Get number of users using a driver; Include DMI and lspci information in HWDB submissions",
+ "dateLastCrawled": "2018-03-10T13:58:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";,
+ "name": "Ubuntu Bugs in Launchpad",
+ "url": "https://launchpad.net/~ubuntu-bugs";,
+ "urlPingSuffix": "DevEx,5339.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "snippet": "The purpose of this team is to have a mailing list that receives all Ubuntu bug mail. If you wish to receive copies of all bug traffic, instead subscribe to the ...",
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";,
+ "name": "BugExpiry - Launchpad Help",
+ "url": "https://help.launchpad.net/BugExpiry";,
+ "urlPingSuffix": "DevEx,5354.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/BugExpiry";,
+ "snippet": "Want to know more about Bug Statuses? Check out our BugStatuses page. What is bug expiry? Today, projects and distributions have the option of allowing old ...",
+ "dateLastCrawled": "2018-01-13T13:58:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";,
+ "name": "gdiskdump project files : gdiskdump",
+ "url": "https://launchpad.net/gdiskdump/+download";,
+ "urlPingSuffix": "DevEx,5367.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gdiskdump/+download";,
+ "snippet": "added advanced settings and estimated Time for process to finish. bug fixes for some languages.",
+ "dateLastCrawled": "2018-02-19T11:52:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";,
+ "name": "Bug #1747091 “epicsTimeGetEvent() / generalTime bug ...",
+ "url": "https://code.launchpad.net/bugs/1747091";,
+ "urlPingSuffix": "DevEx,5379.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://code.launchpad.net/bugs/1747091";,
+ "snippet": "When changes in epicsGeneralTime.c were made (fetch time provider's eventTime into a local copy) an inconsistency resulted.",
+ "dateLastCrawled": "2018-02-04T02:49:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ },
+ "rankingResponse": {
+ "mainline": {
+ "items": [
+ {
+ "answerType": "WebPages",
+ "resultIndex": 0,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 1,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 2,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 3,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 4,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 5,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 6,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 7,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 8,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 9,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 10,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 11,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 12,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 13,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 14,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";
+ }
+ }
+ ]
+ }
+ }
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-bugs-2.json 2018-03-16 21:18:39 +0000
@@ -0,0 +1,384 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=4C3A184771914F6BB81F8541E7EE7695&CID=13045103CBA569B80D475AB7CA03687C&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=4C3A184771914F6BB81F8541E7EE7695&CID=13045103CBA569B80D475AB7CA03687C&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "bug"
+ },
+ "webPages": {
+ "webSearchUrl": "https://www.bing.com/search?q=bug";,
+ "webSearchUrlPingSuffix": "DevEx,5462.1",
+ "totalEstimatedMatches": 87000,
+ "value": [
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";,
+ "name": "Ubuntu Bugs in Launchpad",
+ "url": "https://launchpad.net/~ubuntu-bugs";,
+ "urlPingSuffix": "DevEx,5080.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/~ubuntu-bugs";,
+ "snippet": "The purpose of this team is to have a mailing list that receives all Ubuntu bug mail. If you wish to receive copies of all bug traffic, instead subscribe to the ...",
+ "dateLastCrawled": "2018-03-10T00:32:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";,
+ "name": "GCleaner in Launchpad",
+ "url": "https://launchpad.net/gcleaner";,
+ "urlPingSuffix": "DevEx,5095.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gcleaner";,
+ "snippet": "GCleaner is a beautiful and fast system cleaner for Elementary OS and Ubuntu or based. Writen in Vala, GTK+, Granite and GLib/GIO for the purpose that the users feel ...",
+ "dateLastCrawled": "2018-02-21T13:17:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";,
+ "name": "BugExpiry - Launchpad Help",
+ "url": "https://help.launchpad.net/BugExpiry";,
+ "urlPingSuffix": "DevEx,5110.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/BugExpiry";,
+ "snippet": "Want to know more about Bug Statuses? Check out our BugStatuses page. What is bug expiry? Today, projects and distributions have the option of allowing old ...",
+ "dateLastCrawled": "2018-01-13T13:58:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";,
+ "name": "LongoMatch in Launchpad",
+ "url": "https://launchpad.net/longomatch";,
+ "urlPingSuffix": "DevEx,5125.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/longomatch";,
+ "snippet": "LongoMatch is a sports video analyse tool for coaches to assist them on making live video reports from a match. It creates a database with the most important plays of ...",
+ "dateLastCrawled": "2018-02-16T21:54:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";,
+ "name": "Question #232632 : Questions : OpenStack Heat",
+ "url": "https://answers.launchpad.net/heat/+question/232632";,
+ "urlPingSuffix": "DevEx,5140.1",
+ "datePublished": "2013-07-18T00:00:00.0000000",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://answers.launchpad.net/heat/+question/232632";,
+ "snippet": "Using grizzly version(2013.1.2) from heat/openstack. Failed to create stack, got error: 'NoneType' object has no attribute 'rstrip'. See below my runtime ...",
+ "dateLastCrawled": "2018-02-20T23:45:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";,
+ "name": "Sandpad in Launchpad",
+ "url": "https://launchpad.net/sandpad";,
+ "urlPingSuffix": "DevEx,5154.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/sandpad";,
+ "snippet": "Sandpad is an standalone wlua application for making quick scratch programs and sandboxes in Lua. It uses IUP 3 for the user interface.",
+ "dateLastCrawled": "2018-03-03T17:54:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";,
+ "name": "Eventum in Launchpad",
+ "url": "https://launchpad.net/eventum";,
+ "urlPingSuffix": "DevEx,5169.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/eventum";,
+ "snippet": "Eventum is a user-friendly and flexible issue tracking system that can be used by a support department to track incoming technical support requests, or by a software ...",
+ "dateLastCrawled": "2018-03-07T19:52:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";,
+ "name": "Inkscape 0.48.2 \"0.48.2\" - Launchpad",
+ "url": "https://launchpad.net/inkscape/+milestone/0.48.2";,
+ "urlPingSuffix": "DevEx,5182.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/inkscape/+milestone/0.48.2";,
+ "snippet": "After you've downloaded a file, you can verify its authenticity using its MD5 sum or signature. (How do I verify a download?",
+ "dateLastCrawled": "2018-02-24T19:01:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";,
+ "name": "Code Hosting - Launchpad tour",
+ "url": "https://launchpad.net/+tour/branch-hosting-tracking";,
+ "urlPingSuffix": "DevEx,5195.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/+tour/branch-hosting-tracking";,
+ "snippet": "Code hosting and review. Launchpad and Bazaar make it easy for anyone to get your project's code, make their own changes with full version control, and then propose ...",
+ "dateLastCrawled": "2018-03-10T23:01:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";,
+ "name": "gdiskdump project files : gdiskdump",
+ "url": "https://launchpad.net/gdiskdump/+download";,
+ "urlPingSuffix": "DevEx,5208.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/gdiskdump/+download";,
+ "snippet": "added advanced settings and estimated Time for process to finish. bug fixes for some languages.",
+ "dateLastCrawled": "2018-02-19T11:52:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";,
+ "name": "Bug #1747091 “epicsTimeGetEvent() / generalTime bug ...",
+ "url": "https://code.launchpad.net/bugs/1747091";,
+ "urlPingSuffix": "DevEx,5220.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://code.launchpad.net/bugs/1747091";,
+ "snippet": "When changes in epicsGeneralTime.c were made (fetch time provider's eventTime into a local copy) an inconsistency resulted.",
+ "dateLastCrawled": "2018-02-04T02:49:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";,
+ "name": "Bug #96441 “timer updates use too much CPU” : Bugs : Jokosher",
+ "url": "https://launchpad.net/jokosher/+bug/96441";,
+ "urlPingSuffix": "DevEx,5232.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/jokosher/+bug/96441";,
+ "snippet": "The information about this bug in Launchpad is automatically pulled daily from the remote bug. This information was last pulled ...",
+ "dateLastCrawled": "2018-02-17T16:53:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";,
+ "name": "Series 17.10 : Mahara",
+ "url": "https://launchpad.net/mahara/17.10";,
+ "urlPingSuffix": "DevEx,5247.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/mahara/17.10";,
+ "snippet": "All bugs Latest bugs reported. Bug #1752688: MariaDB fails to upgrade - unable to CAST as JSON Reported on 2018-03-01 Bug #1752442: Problems with group forums / topics",
+ "dateLastCrawled": "2018-03-12T13:48:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";,
+ "name": "sap in Launchpad",
+ "url": "https://launchpad.net/sap+";,
+ "urlPingSuffix": "DevEx,5260.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/sap+";,
+ "snippet": "Sap is a simple audio player written in Vala and utilizing gstreamer for audio playback and ncurses for user interaction.",
+ "dateLastCrawled": "2018-03-12T05:18:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";,
+ "name": "Code/BugAndBlueprintLinks - Launchpad Help",
+ "url": "https://help.launchpad.net/Code/BugAndBlueprintLinks";,
+ "urlPingSuffix": "DevEx,5275.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://help.launchpad.net/Code/BugAndBlueprintLinks";,
+ "snippet": "Linking code to bug reports and blueprints. Launchpad is much like a fancy china dinner service: you can get a great deal of use and contentment from just one or two ...",
+ "dateLastCrawled": "2018-01-26T14:46:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.15";,
+ "name": "One Hundred Papercuts in Launchpad",
+ "url": "https://launchpad.net/hundredpapercuts";,
+ "urlPingSuffix": "DevEx,5289.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/hundredpapercuts";,
+ "snippet": "Downloads. One Hundred Papercuts does not have any download files registered with Launchpad. •",
+ "dateLastCrawled": "2018-03-01T20:05:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.16";,
+ "name": "Little Software Stats in Launchpad",
+ "url": "https://launchpad.net/lilsoftstats/";,
+ "urlPingSuffix": "DevEx,5304.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/lilsoftstats";,
+ "snippet": "Little Software Stats is the first free and open source program that will allow software developers to keep track of how their software is being used.",
+ "dateLastCrawled": "2018-03-07T21:46:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.17";,
+ "name": "Take the tour - Launchpad tour",
+ "url": "https://launchpad.net/+tour";,
+ "urlPingSuffix": "DevEx,5317.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/+tour";,
+ "snippet": "Launchpad helps people to work together on free software by making it easy to share code, bug reports, translations and ideas.",
+ "dateLastCrawled": "2018-02-25T04:37:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.18";,
+ "name": "EOEC in Launchpad",
+ "url": "https://launchpad.net/eoec";,
+ "urlPingSuffix": "DevEx,5332.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/eoec";,
+ "snippet": "Barcode example published as standalone extension on 2009-01-15 After adding a number of extra features to the Barcode example of EuroOffice ... EOEC ...",
+ "dateLastCrawled": "2018-03-09T21:39:00.0000000Z",
+ "fixedPosition": false
+ },
+ {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.19";,
+ "name": "Bug #1210747 “taskdef name not specified” : Bugs : Radkarte",
+ "url": "https://launchpad.net/radkarte/+bug/1210747";,
+ "urlPingSuffix": "DevEx,5345.1",
+ "isFamilyFriendly": true,
+ "displayUrl": "https://launchpad.net/radkarte/+bug/1210747";,
+ "snippet": "Sorry for the late response, somehow the notification mail did not work. I use the \"ant-contrib\" package for the ID calculation to prepare the build-file for easy ...",
+ "dateLastCrawled": "2018-01-11T22:23:00.0000000Z",
+ "fixedPosition": false
+ }
+ ]
+ },
+ "rankingResponse": {
+ "mainline": {
+ "items": [
+ {
+ "answerType": "WebPages",
+ "resultIndex": 0,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.0";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 1,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.1";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 2,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.2";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 3,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.3";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 4,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.4";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 5,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.5";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 6,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.6";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 7,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.7";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 8,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.8";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 9,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.9";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 10,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.10";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 11,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.11";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 12,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.12";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 13,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.13";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 14,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.14";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 15,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.15";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 16,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.16";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 17,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.17";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 18,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.18";
+ }
+ },
+ {
+ "answerType": "WebPages",
+ "resultIndex": 19,
+ "value": {
+ "id": "https://api.cognitive.microsoft.com/api/v7/#WebPages.19";
+ }
+ }
+ ]
+ }
+ }
+}
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-incomplete-response.json 2018-03-16 21:18:39 +0000
@@ -0,0 +1,1 @@
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-mapping.txt 2018-03-16 21:18:39 +0000
@@ -0,0 +1,24 @@
+# This file defines a mapping of Bing search service URLs to the JSON
+# files that should be returned by them.
+#
+# The format is 'url JSONfile'. Blank lines and lines starting with '#'
+# are ignored.
+#
+# The special URL, '*', is returned for all un-mapped URLs.
+
+* bingsearchservice-no-results.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=bug&offset=0 bingsearchservice-bugs-1.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=bug&offset=20 bingsearchservice-bugs-2.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=launchpad&offset=0 bingsearchservice-bugs-1.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=launchpad&offset=20 bingsearchservice-bugs-2.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=gnomebaker&offset=0 bingsearchservice-incomplete-response.json
+
+/bingcustomsearch/v7.0/search?q=bug&customconfig=1234567890&mkt=en-US&safesearch=Moderate&count=20&q=no-meaningful&offset=0 bingsearchservice-no-meaningful-results.json
+
+# This stub service is also used to impersonate the Blog feed
+/blog-feed blog.launchpad.net-feed.json
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-no-meaningful-results.json 2018-03-16 21:18:39 +0000
@@ -0,0 +1,5 @@
+{
+ "_type": "SearchResponse",
+ "webPages": {}
+}
+
=== added file 'lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json'
--- lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/data/bingsearchservice-no-results.json 2018-03-16 21:18:39 +0000
@@ -0,0 +1,12 @@
+{
+ "_type": "SearchResponse",
+ "instrumentation": {
+ "pingUrlBase": "https://www.bingapis.com/api/ping?IG=74D50DC66A774E74AF486941DDF2C89C&CID=2023A1AFC6086AE52F08AA1BC7476B1F&ID=";,
+ "pageLoadPingUrl": "https://www.bingapis.com/api/ping/pageload?IG=74D50DC66A774E74AF486941DDF2C89C&CID=2023A1AFC6086AE52F08AA1BC7476B1F&Type=Event.CPT&DATA=0";
+ },
+ "queryContext": {
+ "originalQuery": "AELymgURIr4plE6V5qUaesxj1S8kUSFxCVrVLNU_OeCogh9Q7W5be6lEGMcbb0q6WTDgLL7zsnlnYGLvVrsdxgx3AamFm0M6ARaxerSLvSf-1JQHrOLuhsQ"
+ },
+ "rankingResponse": {}
+}
+
=== added file 'lib/lp/services/sitesearch/tests/test_bing.py'
--- lib/lp/services/sitesearch/tests/test_bing.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bing.py 2018-03-16 21:18:39 +0000
@@ -0,0 +1,89 @@
+# Copyright 2011-2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Test the bing search service."""
+
+__metaclass__ = type
+
+from contextlib import contextmanager
+
+from requests.exceptions import (
+ ConnectionError,
+ HTTPError,
+ )
+
+from lp.services.sitesearch import BingSearchService
+from lp.services.sitesearch.interfaces import BingResponseError
+from lp.services.timeout import TimeoutError
+from lp.testing import TestCase
+from lp.testing.layers import LaunchpadFunctionalLayer
+
+
+@contextmanager
+def urlfetch_exception(test_error, *args):
+ """Raise an error during the execution of urlfetch.
+
+ This function replaces urlfetch() with a function that
+ raises an error.
+ """
+
+ def raise_exception(url):
+ raise test_error(*args)
+
+ from lp.services import timeout
+ original_urlfetch = timeout.urlfetch
+ timeout.urlfetch = raise_exception
+ try:
+ yield
+ finally:
+ timeout.urlfetch = original_urlfetch
+
+
+class TestBingSearchService(TestCase):
+ """Test GoogleSearchService."""
+
+ layer = LaunchpadFunctionalLayer
+
+ def setUp(self):
+ super(TestBingSearchService, self).setUp()
+ self.search_service = BingSearchService()
+
+ def test_search_converts_HTTPError(self):
+ # The method converts HTTPError to BingResponseError.
+ args = ('url', 500, 'oops', {}, None)
+ with urlfetch_exception(HTTPError, *args):
+ self.assertRaises(
+ BingResponseError, self.search_service.search, 'fnord')
+
+ def test_search_converts_ConnectionError(self):
+ # The method converts ConnectionError to BingResponseError.
+ with urlfetch_exception(ConnectionError, 'oops'):
+ self.assertRaises(
+ BingResponseError, self.search_service.search, 'fnord')
+
+ def test_search_converts_TimeoutError(self):
+ # The method converts TimeoutError to BingResponseError.
+ with urlfetch_exception(TimeoutError, 'oops'):
+ self.assertRaises(
+ BingResponseError, self.search_service.search, 'fnord')
+
+ def test_parse_bing_reponse_TypeError(self):
+ # The method converts TypeError to BingResponseError.
+ with urlfetch_exception(TypeError, 'oops'):
+ self.assertRaises(
+ BingResponseError,
+ self.search_service._parse_bing_response, None)
+
+ def test_parse_bing_reponse_ValueError(self):
+ # The method converts ValueError to BingResponseError.
+ with urlfetch_exception(ValueError, 'oops'):
+ self.assertRaises(
+ BingResponseError,
+ self.search_service._parse_bing_response, '')
+
+ def test_parse_bing_reponse_KeyError(self):
+ # The method converts KeyError to BingResponseError.
+ with urlfetch_exception(KeyError, 'oops'):
+ self.assertRaises(
+ BingResponseError,
+ self.search_service._parse_bing_response, '{}')
=== added file 'lib/lp/services/sitesearch/tests/test_bingharness.py'
--- lib/lp/services/sitesearch/tests/test_bingharness.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bingharness.py 2018-03-16 21:18:39 +0000
@@ -0,0 +1,10 @@
+# Copyright 2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import doctest
+
+
+def test_suite():
+ return doctest.DocTestSuite(
+ 'lp.services.sitesearch.tests.bingserviceharness',
+ optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)
=== added file 'lib/lp/services/sitesearch/tests/test_bingservice.py'
--- lib/lp/services/sitesearch/tests/test_bingservice.py 1970-01-01 00:00:00 +0000
+++ lib/lp/services/sitesearch/tests/test_bingservice.py 2018-03-16 21:18:39 +0000
@@ -0,0 +1,53 @@
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""
+Unit tests for the Bing test service stub.
+"""
+
+__metaclass__ = type
+
+
+import errno
+import os
+import unittest
+
+from lp.services.pidfile import pidfile_path
+from lp.services.sitesearch import bingtestservice
+
+
+class TestServiceUtilities(unittest.TestCase):
+ """Test the service's supporting functions."""
+
+ def test_stale_pid_file_cleanup(self):
+ """The service should be able to clean up invalid PID files."""
+ bogus_pid = 9999999
+ self.assertFalse(
+ process_exists(bogus_pid),
+ "There is already a process with PID '%d'." % bogus_pid)
+
+ # Create a stale/bogus PID file.
+ filepath = pidfile_path(bingtestservice.service_name)
+ pidfile = file(filepath, 'w')
+ pidfile.write(str(bogus_pid))
+ pidfile.close()
+
+ # The PID clean-up code should silently remove the file and return.
+ bingtestservice.kill_running_process()
+ self.assertFalse(
+ os.path.exists(filepath),
+ "The PID file '%s' should have been deleted." % filepath)
+
+
+def process_exists(pid):
+ """Return True if the specified process already exists."""
+ try:
+ os.kill(pid, 0)
+ except os.error as err:
+ if err.errno == errno.ESRCH:
+ # All is well - the process doesn't exist.
+ return False
+ else:
+ # We got a strange OSError, which we'll pass upwards.
+ raise
+ return True
=== modified file 'lib/lp/testing/layers.py'
--- lib/lp/testing/layers.py 2018-03-16 14:55:41 +0000
+++ lib/lp/testing/layers.py 2018-03-16 21:18:39 +0000
@@ -1,4 +1,4 @@
-# Copyright 2009-2016 Canonical Ltd. This software is licensed under the
+# Copyright 2009-2018 Canonical Ltd. This software is licensed under the
# GNU Affero General Public License version 3 (see the file LICENSE).
"""Layers used by Launchpad tests.
@@ -23,6 +23,8 @@
'AppServerLayer',
'AuditorLayer',
'BaseLayer',
+ 'BingLaunchpadFunctionalLayer',
+ 'BingServiceLayer',
'DatabaseFunctionalLayer',
'DatabaseLayer',
'FunctionalLayer',
@@ -126,6 +128,9 @@
from lp.services.osutils import kill_by_pidfile
from lp.services.rabbit.server import RabbitServer
from lp.services.scripts import execute_zcml_for_scripts
+from lp.services.sitesearch.tests.bingserviceharness import (
+ BingServiceTestSetup,
+ )
from lp.services.sitesearch.tests.googleserviceharness import (
GoogleServiceTestSetup,
)
@@ -1259,6 +1264,31 @@
pass
+class BingServiceLayer(BaseLayer):
+ """Tests for Bing web service integration."""
+
+ @classmethod
+ def setUp(cls):
+ bing = BingServiceTestSetup()
+ bing.setUp()
+
+ @classmethod
+ def tearDown(cls):
+ BingServiceTestSetup().tearDown()
+
+ @classmethod
+ def testSetUp(self):
+ # We need to override BaseLayer.testSetUp(), or else we will
+ # get a LayerIsolationError.
+ pass
+
+ @classmethod
+ def testTearDown(self):
+ # We need to override BaseLayer.testTearDown(), or else we will
+ # get a LayerIsolationError.
+ pass
+
+
class DatabaseFunctionalLayer(DatabaseLayer, FunctionalLayer):
"""Provides the database and the Zope3 application server environment."""
@@ -1383,6 +1413,31 @@
pass
+class BingLaunchpadFunctionalLayer(LaunchpadFunctionalLayer,
+ BingServiceLayer):
+ """Provides Bing service in addition to LaunchpadFunctionalLayer."""
+
+ @classmethod
+ @profiled
+ def setUp(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def tearDown(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def testSetUp(cls):
+ pass
+
+ @classmethod
+ @profiled
+ def testTearDown(cls):
+ pass
+
+
class ZopelessDatabaseLayer(ZopelessLayer, DatabaseLayer):
"""Testing layer for unit tests with no need for librarian.
@@ -1541,7 +1596,7 @@
return self.request._orig_env
-class PageTestLayer(LaunchpadFunctionalLayer, GoogleServiceLayer):
+class PageTestLayer(LaunchpadFunctionalLayer, BingServiceLayer):
"""Environment for page tests.
"""
=== modified file 'setup.py'
--- setup.py 2018-03-16 15:14:34 +0000
+++ setup.py 2018-03-16 21:18:39 +0000
@@ -284,6 +284,8 @@
},
entry_points=dict(
console_scripts=[ # `console_scripts` is a magic name to setuptools
+ 'bingtestservice = '
+ 'lp.services.sitesearch.bingtestservice:main',
'build-twisted-plugin-cache = '
'lp.services.twistedsupport.plugincache:main',
'combine-css = lp.scripts.utilities.js.combinecss:main',
_______________________________________________
Mailing list: https://launchpad.net/~launchpad-reviewers
Post to : launchpad-reviewers@lists.launchpad.net
Unsubscribe : https://launchpad.net/~launchpad-reviewers
More help : https://help.launchpad.net/ListHelp
