This is an automated email from the ASF dual-hosted git repository. brondsem pushed a commit to branch db/8568 in repository https://gitbox.apache.org/repos/asf/allura.git
commit d20babd26a7ff0abdb0113263218ff66e0a010e5 Author: Dave Brondsema <[email protected]> AuthorDate: Wed Oct 2 11:52:41 2024 -0400 [#8568] support basic auth to solr --- Allura/allura/command/show_models.py | 10 ++++++++-- Allura/allura/lib/app_globals.py | 13 +++++++++++-- Allura/allura/lib/solr.py | 22 ++++++++++++++++------ Allura/allura/tasks/index_tasks.py | 22 +++++++++++++--------- Allura/allura/tests/test_commands.py | 4 ++-- Allura/allura/tests/unit/test_solr.py | 11 +++++++---- Allura/development.ini | 9 +++++++-- 7 files changed, 64 insertions(+), 27 deletions(-) diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py index 7bffebffa..b5d9d4300 100644 --- a/Allura/allura/command/show_models.py +++ b/Allura/allura/command/show_models.py @@ -79,6 +79,8 @@ class ReindexCommand(base.Command): 'which are needed for some markdown macros to run properly') parser.add_option('--solr-hosts', dest='solr_hosts', help='Override the solr host(s) to post to. Comma-separated list of solr server URLs') + parser.add_option('--solr-creds', dest='solr_creds', + help='Creds for the solr host(s). Comma-separated list of user:pwd strings') parser.add_option( '--max-chunk', dest='max_chunk', type=int, default=100 * 1000, help='Max number of artifacts to index in one Solr update command') @@ -148,9 +150,13 @@ class ReindexCommand(base.Command): @property def add_artifact_kwargs(self): + kwargs = {} if self.options.solr_hosts: - return {'solr_hosts': self.options.solr_hosts.split(',')} - return {} + kwargs['solr_hosts'] = self.options.solr_hosts.split(',') + if self.options.solr_creds: + kwargs['solr_creds'] = [cred.split(':') + for cred in self.options.solr_creds.split(',')] + return kwargs def _chunked_add_artifacts(self, ref_ids): # ref_ids contains solr index ids which can easily be over diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py index d4bf931a4..fc6bfbba9 100644 --- a/Allura/allura/lib/app_globals.py +++ b/Allura/allura/lib/app_globals.py @@ -203,13 +203,22 @@ class Globals: self.solr_server = aslist(config.get('solr.server'), ',') # skip empty strings in case of extra commas self.solr_server = [s for s in self.solr_server if s] + push_auths = zip(aslist(config.get('solr.user'), ','), + aslist(config.get('solr.pass'), ',')) self.solr_query_server = config.get('solr.query_server') + query_auth = (config.get('solr.query_user'), config.get('solr.query_pass')) if self.solr_server: self.solr = make_solr_from_config( - self.solr_server, self.solr_query_server) + self.solr_server, self.solr_query_server, + push_servers_auths=push_auths, + query_server_auth=query_auth, + ) self.solr_short_timeout = make_solr_from_config( self.solr_server, self.solr_query_server, - timeout=int(config.get('solr.short_timeout', 10))) + push_servers_auths=push_auths, + query_server_auth=query_auth, + timeout=int(config.get('solr.short_timeout', 10)), + ) else: # pragma no cover log.warning('Solr config not set; using in-memory MockSOLR') self.solr = self.solr_short_timeout = MockSOLR() diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py index 7635b36d6..012fb728a 100644 --- a/Allura/allura/lib/solr.py +++ b/Allura/allura/lib/solr.py @@ -14,10 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +from __future__ import annotations import json import logging +from itertools import zip_longest +from collections.abc import Iterable from tg import config from webob.exc import HTTPRequestEntityTooLarge @@ -61,7 +63,10 @@ def escape_solr_arg(term): return term -def make_solr_from_config(push_servers, query_server=None, **kwargs): +def make_solr_from_config(push_servers: Iterable[str], query_server: str|None=None, + push_servers_auths: Iterable[tuple[str, str] | None] = (), + query_server_auth: tuple[str, str] | None = None, + **kwargs): """ Make a :class:`Solr <Solr>` instance from config defaults. Use `**kwargs` to override any value @@ -72,7 +77,7 @@ def make_solr_from_config(push_servers, query_server=None, **kwargs): timeout=int(config.get('solr.long_timeout', 60)), ) solr_kwargs.update(kwargs) - return Solr(push_servers, query_server, **solr_kwargs) + return Solr(push_servers, query_server, push_servers_auths, query_server_auth, **solr_kwargs) class Solr: @@ -87,13 +92,18 @@ class Solr: unless explicitly overridden. """ - def __init__(self, push_servers, query_server=None, + def __init__(self, push_servers: Iterable[str], query_server: str|None = None, + push_servers_auths: Iterable[tuple[str, str] | None] = (), + query_server_auth: tuple[str, str] | None = None, commit=True, commitWithin=None, **kw): - self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers] + self.push_pool = [pysolr.Solr(s, auth=auth, **kw) + for s, auth in zip_longest(push_servers, push_servers_auths)] if query_server: - self.query_server = pysolr.Solr(query_server, **kw) + self.query_server = pysolr.Solr(query_server, auth=query_server_auth, **kw) else: self.query_server = self.push_pool[0] + if query_server_auth: + self.query_server.auth = query_server_auth self._commit = commit self.commitWithin = commitWithin diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py index aec34c917..d907b3319 100644 --- a/Allura/allura/tasks/index_tasks.py +++ b/Allura/allura/tasks/index_tasks.py @@ -18,6 +18,7 @@ from __future__ import annotations import sys import logging +from collections.abc import Iterable from contextlib import contextmanager import typing @@ -38,12 +39,12 @@ if typing.TYPE_CHECKING: log = logging.getLogger(__name__) -def __get_solr(solr_hosts=None): - return make_solr_from_config(solr_hosts) if solr_hosts else g.solr +def __get_solr(solr_hosts=None, solr_creds=()): + return make_solr_from_config(solr_hosts, push_servers_auths=solr_creds) if solr_hosts else g.solr -def __add_objects(objects, solr_hosts=None): - solr_instance = __get_solr(solr_hosts) +def __add_objects(objects, solr_hosts=None, solr_creds=()): + solr_instance = __get_solr(solr_hosts, solr_creds) solr_instance.add([obj.solarize() for obj in objects]) @@ -94,16 +95,19 @@ def del_users(user_solr_ids): @task -def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): +def add_artifacts(ref_ids, update_solr=True, update_refs=True, + solr_hosts: Iterable[str] = (), + solr_creds: Iterable[tuple[str, str]] = (), + ): ''' Add the referenced artifacts to SOLR and shortlinks. - - :param solr_hosts: a list of solr hosts to use instead of the defaults - :type solr_hosts: [str] ''' from allura import model as M from allura.lib.search import find_shortlinks + # task params end up as instrumented lists, need to make this a list of plain tuples + solr_creds = [tuple(cred) for cred in solr_creds] + exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): @@ -146,7 +150,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): log.info("Solr.add raised HTTPRequestEntityTooLarge but there is only one artifact. Raising exception.") raise - _add_artifact(__get_solr(solr_hosts), solr_updates) + _add_artifact(__get_solr(solr_hosts, solr_creds), solr_updates) if len(exceptions) == 1: raise exceptions[0][1].with_traceback(exceptions[0][2]) diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py index c228835af..994b44d33 100644 --- a/Allura/allura/tests/test_commands.py +++ b/Allura/allura/tests/test_commands.py @@ -532,7 +532,7 @@ class TestReindexCommand: @patch('allura.command.show_models.add_artifacts') def test_chunked_add_artifacts(self, add_artifacts): cmd = show_models.ReindexCommand('reindex') - cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None) + cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None, solr_creds='') ref_ids = list(range(10 * 1000 * 2 + 20)) cmd._chunked_add_artifacts(ref_ids) assert len(add_artifacts.post.call_args_list) == 3 @@ -575,7 +575,7 @@ class TestReindexCommand: raise pymongo.errors.InvalidDocument("Cannot encode object...") add_artifacts.post.side_effect = on_post cmd = show_models.ReindexCommand('reindex') - cmd.options = Mock(ming_config=None) + cmd.options = Mock(ming_config=None, solr_creds='') with td.raises(pymongo.errors.InvalidDocument): cmd._post_add_artifacts(list(range(5))) diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py index 384a985bf..7dd7328e6 100644 --- a/Allura/allura/tests/unit/test_solr.py +++ b/Allura/allura/tests/unit/test_solr.py @@ -39,15 +39,18 @@ class TestSolr(unittest.TestCase): @mock.patch('allura.lib.solr.pysolr') def test_init(self, pysolr): servers = ['server1', 'server2'] - solr = Solr(servers, commit=False, commitWithin='10000') - calls = [mock.call('server1'), mock.call('server2')] + auths = [('u', 'pwd'),] + solr = Solr(servers, push_servers_auths=auths, commit=False, commitWithin='10000') + calls = [mock.call('server1', auth=('u', 'pwd')), mock.call('server2', auth=None)] pysolr.Solr.assert_has_calls(calls) assert len(solr.push_pool) == 2 pysolr.reset_mock() solr = Solr(servers, 'server3', commit=False, commitWithin='10000') - calls = [mock.call('server1'), mock.call('server2'), - mock.call('server3')] + calls = [mock.call('server1', auth=None), + mock.call('server2', auth=None), + mock.call('server3', auth=None), + ] pysolr.Solr.assert_has_calls(calls) assert len(solr.push_pool) == 2 diff --git a/Allura/development.ini b/Allura/development.ini index b91356088..9d64bd910 100644 --- a/Allura/development.ini +++ b/Allura/development.ini @@ -571,10 +571,15 @@ stats.sample_rate = 1 ; number of seconds to sleep between checking for new tasks monq.poll_interval=2 -; SOLR setup +; SOLR setup. This can be a comma-separated list, to index into multiple locations solr.server = http://localhost:8983/solr/allura -; Alternate server to use just for querying +# auth for solr, if needed. These can be a list too, to match solr.server +;solr.user = +;solr.pass = +; Alternate server to use just for querying. Single server, not a list. ;solr.query_server = +;solr.query_user = +;solr.query_pass = ; Shorter timeout for search queries (longer timeout for saving to solr) solr.short_timeout = 10 ; commit on every add/delete?
