This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
The following commit(s) were added to refs/heads/master by this push:
new ac2b3d9a5 [#8568] support basic auth to solr
ac2b3d9a5 is described below
commit ac2b3d9a5753c2c071ef6c4ef0f0186d0471b85a
Author: Dave Brondsema <[email protected]>
AuthorDate: Wed Oct 2 11:52:41 2024 -0400
[#8568] support basic auth to solr
---
Allura/allura/command/show_models.py | 10 ++++++++--
Allura/allura/lib/app_globals.py | 13 +++++++++++--
Allura/allura/lib/solr.py | 22 ++++++++++++++++------
Allura/allura/tasks/index_tasks.py | 22 +++++++++++++---------
Allura/allura/tests/test_commands.py | 4 ++--
Allura/allura/tests/unit/test_solr.py | 11 +++++++----
Allura/development.ini | 9 +++++++--
7 files changed, 64 insertions(+), 27 deletions(-)
diff --git a/Allura/allura/command/show_models.py
b/Allura/allura/command/show_models.py
index 7bffebffa..b5d9d4300 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -79,6 +79,8 @@ class ReindexCommand(base.Command):
'which are needed for some markdown macros to run
properly')
parser.add_option('--solr-hosts', dest='solr_hosts',
help='Override the solr host(s) to post to.
Comma-separated list of solr server URLs')
+ parser.add_option('--solr-creds', dest='solr_creds',
+ help='Creds for the solr host(s). Comma-separated list
of user:pwd strings')
parser.add_option(
'--max-chunk', dest='max_chunk', type=int, default=100 * 1000,
help='Max number of artifacts to index in one Solr update command')
@@ -148,9 +150,13 @@ class ReindexCommand(base.Command):
@property
def add_artifact_kwargs(self):
+ kwargs = {}
if self.options.solr_hosts:
- return {'solr_hosts': self.options.solr_hosts.split(',')}
- return {}
+ kwargs['solr_hosts'] = self.options.solr_hosts.split(',')
+ if self.options.solr_creds:
+ kwargs['solr_creds'] = [cred.split(':')
+ for cred in
self.options.solr_creds.split(',')]
+ return kwargs
def _chunked_add_artifacts(self, ref_ids):
# ref_ids contains solr index ids which can easily be over
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index d4bf931a4..fc6bfbba9 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -203,13 +203,22 @@ class Globals:
self.solr_server = aslist(config.get('solr.server'), ',')
# skip empty strings in case of extra commas
self.solr_server = [s for s in self.solr_server if s]
+ push_auths = zip(aslist(config.get('solr.user'), ','),
+ aslist(config.get('solr.pass'), ','))
self.solr_query_server = config.get('solr.query_server')
+ query_auth = (config.get('solr.query_user'),
config.get('solr.query_pass'))
if self.solr_server:
self.solr = make_solr_from_config(
- self.solr_server, self.solr_query_server)
+ self.solr_server, self.solr_query_server,
+ push_servers_auths=push_auths,
+ query_server_auth=query_auth,
+ )
self.solr_short_timeout = make_solr_from_config(
self.solr_server, self.solr_query_server,
- timeout=int(config.get('solr.short_timeout', 10)))
+ push_servers_auths=push_auths,
+ query_server_auth=query_auth,
+ timeout=int(config.get('solr.short_timeout', 10)),
+ )
else: # pragma no cover
log.warning('Solr config not set; using in-memory MockSOLR')
self.solr = self.solr_short_timeout = MockSOLR()
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index 7635b36d6..012fb728a 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -14,10 +14,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+from __future__ import annotations
import json
import logging
+from itertools import zip_longest
+from collections.abc import Iterable
from tg import config
from webob.exc import HTTPRequestEntityTooLarge
@@ -61,7 +63,10 @@ def escape_solr_arg(term):
return term
-def make_solr_from_config(push_servers, query_server=None, **kwargs):
+def make_solr_from_config(push_servers: Iterable[str], query_server:
str|None=None,
+ push_servers_auths: Iterable[tuple[str, str] | None]
= (),
+ query_server_auth: tuple[str, str] | None = None,
+ **kwargs):
"""
Make a :class:`Solr <Solr>` instance from config defaults. Use
`**kwargs` to override any value
@@ -72,7 +77,7 @@ def make_solr_from_config(push_servers, query_server=None,
**kwargs):
timeout=int(config.get('solr.long_timeout', 60)),
)
solr_kwargs.update(kwargs)
- return Solr(push_servers, query_server, **solr_kwargs)
+ return Solr(push_servers, query_server, push_servers_auths,
query_server_auth, **solr_kwargs)
class Solr:
@@ -87,13 +92,18 @@ class Solr:
unless explicitly overridden.
"""
- def __init__(self, push_servers, query_server=None,
+ def __init__(self, push_servers: Iterable[str], query_server: str|None =
None,
+ push_servers_auths: Iterable[tuple[str, str] | None] = (),
+ query_server_auth: tuple[str, str] | None = None,
commit=True, commitWithin=None, **kw):
- self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
+ self.push_pool = [pysolr.Solr(s, auth=auth, **kw)
+ for s, auth in zip_longest(push_servers,
push_servers_auths)]
if query_server:
- self.query_server = pysolr.Solr(query_server, **kw)
+ self.query_server = pysolr.Solr(query_server,
auth=query_server_auth, **kw)
else:
self.query_server = self.push_pool[0]
+ if query_server_auth:
+ self.query_server.auth = query_server_auth
self._commit = commit
self.commitWithin = commitWithin
diff --git a/Allura/allura/tasks/index_tasks.py
b/Allura/allura/tasks/index_tasks.py
index aec34c917..d907b3319 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import sys
import logging
+from collections.abc import Iterable
from contextlib import contextmanager
import typing
@@ -38,12 +39,12 @@ if typing.TYPE_CHECKING:
log = logging.getLogger(__name__)
-def __get_solr(solr_hosts=None):
- return make_solr_from_config(solr_hosts) if solr_hosts else g.solr
+def __get_solr(solr_hosts=None, solr_creds=()):
+ return make_solr_from_config(solr_hosts, push_servers_auths=solr_creds) if
solr_hosts else g.solr
-def __add_objects(objects, solr_hosts=None):
- solr_instance = __get_solr(solr_hosts)
+def __add_objects(objects, solr_hosts=None, solr_creds=()):
+ solr_instance = __get_solr(solr_hosts, solr_creds)
solr_instance.add([obj.solarize() for obj in objects])
@@ -94,16 +95,19 @@ def del_users(user_solr_ids):
@task
-def add_artifacts(ref_ids, update_solr=True, update_refs=True,
solr_hosts=None):
+def add_artifacts(ref_ids, update_solr=True, update_refs=True,
+ solr_hosts: Iterable[str] = (),
+ solr_creds: Iterable[tuple[str, str]] = (),
+ ):
'''
Add the referenced artifacts to SOLR and shortlinks.
-
- :param solr_hosts: a list of solr hosts to use instead of the defaults
- :type solr_hosts: [str]
'''
from allura import model as M
from allura.lib.search import find_shortlinks
+ # task params end up as instrumented lists, need to make this a list of
plain tuples
+ solr_creds = [tuple(cred) for cred in solr_creds]
+
exceptions = []
solr_updates = []
with _indexing_disabled(M.session.artifact_orm_session._get()):
@@ -146,7 +150,7 @@ def add_artifacts(ref_ids, update_solr=True,
update_refs=True, solr_hosts=None):
log.info("Solr.add raised HTTPRequestEntityTooLarge but
there is only one artifact. Raising exception.")
raise
- _add_artifact(__get_solr(solr_hosts), solr_updates)
+ _add_artifact(__get_solr(solr_hosts, solr_creds), solr_updates)
if len(exceptions) == 1:
raise exceptions[0][1].with_traceback(exceptions[0][2])
diff --git a/Allura/allura/tests/test_commands.py
b/Allura/allura/tests/test_commands.py
index c228835af..994b44d33 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -532,7 +532,7 @@ class TestReindexCommand:
@patch('allura.command.show_models.add_artifacts')
def test_chunked_add_artifacts(self, add_artifacts):
cmd = show_models.ReindexCommand('reindex')
- cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None)
+ cmd.options = Mock(tasks=True, max_chunk=10 * 1000, ming_config=None,
solr_creds='')
ref_ids = list(range(10 * 1000 * 2 + 20))
cmd._chunked_add_artifacts(ref_ids)
assert len(add_artifacts.post.call_args_list) == 3
@@ -575,7 +575,7 @@ class TestReindexCommand:
raise pymongo.errors.InvalidDocument("Cannot encode object...")
add_artifacts.post.side_effect = on_post
cmd = show_models.ReindexCommand('reindex')
- cmd.options = Mock(ming_config=None)
+ cmd.options = Mock(ming_config=None, solr_creds='')
with td.raises(pymongo.errors.InvalidDocument):
cmd._post_add_artifacts(list(range(5)))
diff --git a/Allura/allura/tests/unit/test_solr.py
b/Allura/allura/tests/unit/test_solr.py
index 384a985bf..7dd7328e6 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -39,15 +39,18 @@ class TestSolr(unittest.TestCase):
@mock.patch('allura.lib.solr.pysolr')
def test_init(self, pysolr):
servers = ['server1', 'server2']
- solr = Solr(servers, commit=False, commitWithin='10000')
- calls = [mock.call('server1'), mock.call('server2')]
+ auths = [('u', 'pwd'),]
+ solr = Solr(servers, push_servers_auths=auths, commit=False,
commitWithin='10000')
+ calls = [mock.call('server1', auth=('u', 'pwd')), mock.call('server2',
auth=None)]
pysolr.Solr.assert_has_calls(calls)
assert len(solr.push_pool) == 2
pysolr.reset_mock()
solr = Solr(servers, 'server3', commit=False, commitWithin='10000')
- calls = [mock.call('server1'), mock.call('server2'),
- mock.call('server3')]
+ calls = [mock.call('server1', auth=None),
+ mock.call('server2', auth=None),
+ mock.call('server3', auth=None),
+ ]
pysolr.Solr.assert_has_calls(calls)
assert len(solr.push_pool) == 2
diff --git a/Allura/development.ini b/Allura/development.ini
index b91356088..9d64bd910 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -571,10 +571,15 @@ stats.sample_rate = 1
; number of seconds to sleep between checking for new tasks
monq.poll_interval=2
-; SOLR setup
+; SOLR setup. This can be a comma-separated list, to index into multiple
locations
solr.server = http://localhost:8983/solr/allura
-; Alternate server to use just for querying
+# auth for solr, if needed. These can be a list too, to match solr.server
+;solr.user =
+;solr.pass =
+; Alternate server to use just for querying. Single server, not a list.
;solr.query_server =
+;solr.query_user =
+;solr.query_pass =
; Shorter timeout for search queries (longer timeout for saving to solr)
solr.short_timeout = 10
; commit on every add/delete?
