The script will be run on a regular basis to get rid of old reports that we don't need. This will improve the performance of the application since the database has grown to a huge size.
The script will remove reports older than thirty days if they have not been referred to by a host other than the Error Reporting Tool. The function details() in views.py will keep trick of the referer when a build failure report is accessed. If there is no referer, we will note that in the database in order to determine how often there is no referer. It's possible that crawlers will access reports with no referer; if that's the case, a lack of a referer should not determine whether we save an older report or not. Fixes [YOCTO #12332] Signed-off-by: Amanda Brindle <amanda.r.brin...@intel.com> --- Post/migrations/0006_buildfailure_referer.py | 19 ++++++++++++++++++ Post/models.py | 10 ++++++++++ Post/purge.py | 29 ++++++++++++++++++++++++++++ Post/views.py | 16 +++++++++++++-- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 Post/migrations/0006_buildfailure_referer.py create mode 100644 Post/purge.py diff --git a/Post/migrations/0006_buildfailure_referer.py b/Post/migrations/0006_buildfailure_referer.py new file mode 100644 index 0000000..5fad048 --- /dev/null +++ b/Post/migrations/0006_buildfailure_referer.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('Post', '0005_build_error_type'), + ] + + operations = [ + migrations.AddField( + model_name='buildfailure', + name='REFERER', + field=models.CharField(default=b'NOT_VISITED', max_length=14, choices=[(b'NO_REFERER', b'no_referer'), (b'OTHER', b'other'), (b'NOT_VISITED', b'not_visited')]), + ), + ] diff --git a/Post/models.py b/Post/models.py index ddf2fc7..fcf53ca 100644 --- a/Post/models.py +++ b/Post/models.py @@ -59,6 +59,16 @@ class BuildFailure(models.Model): ERROR_DETAILS = models.TextField(max_length=int(settings.MAX_UPLOAD_SIZE)) BUILD = models.ForeignKey(Build) LEV_DISTANCE = models.IntegerField(blank=True, null=True) + REFERER_CHOICES = ( + ('NO_REFERER', 'no_referer'), + ('OTHER', 'other'), + ('NOT_VISITED', 'not_visited') + ) + REFERER = models.CharField( + max_length = 14, + choices = REFERER_CHOICES, + default = 'NOT_VISITED' + ) def get_similar_fails(self): if self.LEV_DISTANCE is None: diff --git a/Post/purge.py b/Post/purge.py new file mode 100644 index 0000000..829fada --- /dev/null +++ b/Post/purge.py @@ -0,0 +1,29 @@ +from datetime import datetime +from django.utils import timezone +import os +import sys + +def setup_django(): + import django + # Get access to our Django model + newpath = os.path.abspath(os.path.dirname(__file__)) + '/..' + sys.path.append(newpath) + os.environ['DJANGO_SETTINGS_MODULE'] = 'project.settings' + django.setup() + +def main(): + setup_django() + from Post.models import BuildFailure + items = BuildFailure.objects.all() + now = timezone.now() + for item in items: + if item.REFERER == 'OTHER' or item.REFERER == 'NO_REFERER': + continue + difference = now - item.BUILD.DATE + if difference.days > 30: + item.delete() + +if __name__ == "__main__": + main() + + diff --git a/Post/views.py b/Post/views.py index 7f2cffb..2ea94a7 100644 --- a/Post/views.py +++ b/Post/views.py @@ -22,6 +22,7 @@ from django.http import JsonResponse from django.db.models import Q import json import urllib +from urlparse import urlparse class results_mode(object): LATEST = 0 @@ -256,9 +257,20 @@ def search(request, mode=results_mode.LATEST, **kwargs): def details(request, fail_id): try: - build_failure = BuildFailure.objects.get(id=fail_id) + build_failure = BuildFailure.objects.get(id=fail_id) except ObjectDoesNotExist: - build_failure = None + build_failure = None + try: + referer = urlparse(request.META['HTTP_REFERER']) + referer_hostname = referer.hostname + if referer.port: + referer_hostname += ":" + str(referer.port) + if referer_hostname != request.get_host(): + build_failure.REFERER = 'OTHER' + except KeyError: + # There is no referer + build_failure.REFERER = 'NO_REFERER' + build_failure.save() context = {'detail' : build_failure, 'error_types' : ErrorType } -- 2.7.4 -- _______________________________________________ yocto mailing list yocto@yoctoproject.org https://lists.yoctoproject.org/listinfo/yocto