This is an automated email from the ASF dual-hosted git repository.
dill0wn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/allura.git
The following commit(s) were added to refs/heads/master by this push:
new 07b084ccb fix solr 413 request too big for big batches
07b084ccb is described below
commit 07b084ccbdaa29b4ec774ec68d0bcf1d164e769a
Author: Dillon Walls <[email protected]>
AuthorDate: Thu Nov 2 17:50:51 2023 +0000
fix solr 413 request too big for big batches
---
Allura/allura/lib/solr.py | 10 +++++++++-
Allura/allura/tasks/index_tasks.py | 21 ++++++++++++++++++++-
Allura/allura/tests/unit/test_solr.py | 12 ++++++++++++
3 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index 79c7f61c5..49c76afb3 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -20,8 +20,10 @@ import json
import logging
from tg import config
+from webob.exc import HTTPRequestEntityTooLarge
from paste.deploy.converters import asbool
import pysolr
+from pysolr import SolrError
import six
from allura.lib.helpers import shlex_split
@@ -102,7 +104,13 @@ class Solr:
kw['commitWithin'] = self.commitWithin
responses = []
for solr in self.push_pool:
- responses.append(solr.add(*args, **kw))
+ try:
+ responses.append(solr.add(*args, **kw))
+ except SolrError as e:
+ if '(HTTP 413)' in str(e):
+ raise HTTPRequestEntityTooLarge() from e
+ else:
+ raise
return responses
def delete(self, *args, **kw):
diff --git a/Allura/allura/tasks/index_tasks.py
b/Allura/allura/tasks/index_tasks.py
index 0ce2f7b28..b38aec1a7 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -14,13 +14,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import sys
import logging
from contextlib import contextmanager
+import typing
from tg import app_globals as g
from tg import tmpl_context as c
+from webob.exc import HTTPRequestEntityTooLarge
from allura.lib import helpers as h
from allura.lib.decorators import task
@@ -28,6 +31,9 @@ from allura.lib.exceptions import CompoundError
from allura.lib.solr import make_solr_from_config
import six
+if typing.TYPE_CHECKING:
+ import pysolr
+
log = logging.getLogger(__name__)
@@ -126,7 +132,20 @@ def add_artifacts(ref_ids, update_solr=True,
update_refs=True, solr_hosts=None):
except Exception:
log.error('Error indexing artifact %s', ref._id)
exceptions.append(sys.exc_info())
- __get_solr(solr_hosts).add(solr_updates)
+
+ def _add_artifact(solr: pysolr.Solr, artifacts: list):
+ try:
+ solr.add(artifacts)
+ except HTTPRequestEntityTooLarge:
+ if len(artifacts) > 1:
+ log.warning(f"Solr.add raised HTTPRequestEntityTooLarge.
Splitting {len(artifacts)} updates into two batches.")
+ _add_artifact(solr, artifacts[:len(artifacts) // 2])
+ _add_artifact(solr, artifacts[len(artifacts) // 2:])
+ else:
+ log.info("Solr.add raised HTTPRequestEntityTooLarge but
there is only one artifact. Raising exception.")
+ raise
+
+ _add_artifact(__get_solr(solr_hosts), solr_updates)
if len(exceptions) == 1:
raise exceptions[0][1].with_traceback(exceptions[0][2])
diff --git a/Allura/allura/tests/unit/test_solr.py
b/Allura/allura/tests/unit/test_solr.py
index 43dccb53d..384a985bf 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -19,6 +19,9 @@ import unittest
import mock
from markupsafe import Markup
+from pysolr import SolrError
+import pytest
+from webob.exc import HTTPRequestEntityTooLarge
from allura.lib import helpers as h
from allura.tests import decorators as td
@@ -61,6 +64,15 @@ class TestSolr(unittest.TestCase):
commitWithin='10000', somekw='value')] * 2
pysolr.Solr().add.assert_has_calls(calls)
+ @mock.patch('allura.lib.solr.pysolr')
+ def test_add_too_big(self, pysolr):
+ pysolr.Solr.return_value.add.side_effect = \
+ SolrError("Solr responded with an error (HTTP 413): [Reason:
None]")
+ servers = ['server1', 'server2']
+ solr = Solr(servers, commit=False, commitWithin='10000')
+ with pytest.raises(HTTPRequestEntityTooLarge):
+ solr.add('foo', commit=True, commitWithin=None)
+
@mock.patch('allura.lib.solr.pysolr')
def test_delete(self, pysolr):
servers = ['server1', 'server2']
