New feature added to recreate command line builds that are made when toaster is off. The feature contains a new button on the base template to access a new template. A new model was added to register the information on the builds and generate access links A new form was added to include the option to load specific files This feature uses the value from the variable BB_DEFAULT_EVENTLOG to read the files created by bitbake
Signed-off-by: Marlon Rodriguez Garcia <[email protected]> --- lib/bb/ui/toasterui.py | 2 +- .../orm/migrations/0021_eventlogsimports.py | 22 ++ lib/toaster/orm/models.py | 9 + lib/toaster/toastergui/forms.py | 13 ++ lib/toaster/toastergui/templates/base.html | 3 +- .../templates/command_line_builds.html | 171 ++++++++++++++++ lib/toaster/toastergui/urls.py | 1 + lib/toaster/toastergui/views.py | 188 +++++++++++++++++- 8 files changed, 405 insertions(+), 4 deletions(-) create mode 100644 lib/toaster/orm/migrations/0021_eventlogsimports.py create mode 100644 lib/toaster/toastergui/forms.py create mode 100644 lib/toaster/toastergui/templates/command_line_builds.html diff --git a/lib/bb/ui/toasterui.py b/lib/bb/ui/toasterui.py index ec5bd4f1..6bd21f18 100644 --- a/lib/bb/ui/toasterui.py +++ b/lib/bb/ui/toasterui.py @@ -385,7 +385,7 @@ def main(server, eventHandler, params): main.shutdown = 1 logger.info("ToasterUI build done, brbe: %s", brbe) - continue + break if isinstance(event, (bb.command.CommandCompleted, bb.command.CommandFailed, diff --git a/lib/toaster/orm/migrations/0021_eventlogsimports.py b/lib/toaster/orm/migrations/0021_eventlogsimports.py new file mode 100644 index 00000000..328eb575 --- /dev/null +++ b/lib/toaster/orm/migrations/0021_eventlogsimports.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2.5 on 2023-11-23 18:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0020_models_bigautofield'), + ] + + operations = [ + migrations.CreateModel( + name='EventLogsImports', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('imported', models.BooleanField(default=False)), + ('build_id', models.IntegerField(blank=True, null=True)), + ], + ), + ] diff --git a/lib/toaster/orm/models.py b/lib/toaster/orm/models.py index 1098ad3f..19c96862 100644 --- a/lib/toaster/orm/models.py +++ b/lib/toaster/orm/models.py @@ -1868,6 +1868,15 @@ class Distro(models.Model): def __unicode__(self): return "Distro " + self.name + "(" + self.description + ")" +class EventLogsImports(models.Model): + name = models.CharField(max_length=255) + imported = models.BooleanField(default=False) + build_id = models.IntegerField(blank=True, null=True) + + def __str__(self): + return self.name + + django.db.models.signals.post_save.connect(invalidate_cache) django.db.models.signals.post_delete.connect(invalidate_cache) django.db.models.signals.m2m_changed.connect(invalidate_cache) diff --git a/lib/toaster/toastergui/forms.py b/lib/toaster/toastergui/forms.py new file mode 100644 index 00000000..a87e5391 --- /dev/null +++ b/lib/toaster/toastergui/forms.py @@ -0,0 +1,13 @@ +# +# BitBake Toaster Implementation +# +# Copyright (C) 2013 Intel Corporation +# +# SPDX-License-Identifier: GPL-2.0-only +# + +from django import forms +from django.core.validators import FileExtensionValidator + +class LoadFileForm(forms.Form): + eventlog_file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.json'})) diff --git a/lib/toaster/toastergui/templates/base.html b/lib/toaster/toastergui/templates/base.html index 041448d1..e90be696 100644 --- a/lib/toaster/toastergui/templates/base.html +++ b/lib/toaster/toastergui/templates/base.html @@ -132,7 +132,8 @@ {% if project_enable %} <a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a> {% endif %} - </div> + <a class="btn btn-default navbar-btn navbar-right" id="import_page" style="margin-right: 5px !important" id="import-cmdline-button" href="{% url 'cmdlines' %}">Import command line builds</a> + </div> </div> </nav> diff --git a/lib/toaster/toastergui/templates/command_line_builds.html b/lib/toaster/toastergui/templates/command_line_builds.html new file mode 100644 index 00000000..5b085e33 --- /dev/null +++ b/lib/toaster/toastergui/templates/command_line_builds.html @@ -0,0 +1,171 @@ +{% extends "base.html" %} +{% load projecttags %} +{% load humanize %} + +{% block title %} Import Builds from eventlogs - Toaster {% endblock %} + +{% block pagecontent %} + +<div class="container-fluid"> + <div class="row"> + <div class="col-md-12"> + <div class="page-header"> + <div class="row"> + <div class="col-md-6"> + <h1>Import command line builds</h1> + </div> + {% if import_all %} + <div class="col-md-6"> + <button id="import_all" type="button" class="btn btn-primary navbar-btn navbar-right"> + <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import All + </button> + </div> + {% endif %} + </div> + </div> + {% if messages %} + <div class="row-fluid" id="empty-state-{{table_name}}"> + {% for message in messages %} + <div class="alert alert-danger">{{message}}</div> + {%endfor%} + </div> + {% endif %} + <div class="row"> + <h4 style="margin-left: 15px;"><strong>Import eventlog file</strong></h4> + <form method="POST" enctype="multipart/form-data" action="{% url 'cmdlines' %}"> + {% csrf_token %} + <div class="col-md-6" style="padding-left: 20px;"> + <div class="row"> + <input type="hidden" value="{{dir}}" name="dir"> + <div class="col-md-3"> {{ form.eventlog_file}} </div> + </div> + <div class="row" style="padding-top: 10px;"> + <div class="col-md-6"> + <button id="file_import" type="submit" disabled="disabled" class="btn btn-default navbar-btn" > + <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import + </button> + </div> + </div> + </div> + </form> + </div> + + <div class="row" style="padding-top: 20px;"> + <div class="col-md-6 "> + <h4><strong>Import eventlog files from directory</strong> + <a href="#" data-toggle="tooltip" title="{{dir}}"> + <svg xmlns="http://www.w3.org/2000/svg"; width="16" height="16" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16" data-toggle="tooltip"> + <path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14m0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16"/> + <path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0"/> + </svg> + </a> + </h4> + {% if files %} + <div class="table-responsive"> + <table class="table col-md-4 table-bordered table-hover"> + <thead> + <tr class="row"> + <th scope="col">Name</th> + <th scope="col">Action</th> + </tr> + </thead> + <tbody> + {% for file in files %} + <tr class="row" style="height: 48px;"> + <th scope="row" class="col-md-6" style="vertical-align: middle;"> + <input type="hidden" value="{{file.name}}" name="{{file.name}}">{{file.name}} + </th> + <td class="col-md-6 align-middle" style="vertical-align: middle;"> + {% if file.imported == True and file.build_id is not None %} + <a href="{% url 'builddashboard' file.build_id %}">Build Details</a> + {%else %} + <a onclick="_ajax_update('{{file.name}}', false, '{{dir}}')" data-toggle="tooltip" title="Import File"> + <span class="glyphicon glyphicon-upload" style="font-size: 18px;"></span> + </a> + {%endif%} + </td> + </tr> + {% endfor%} + </tbody> + </table> + </div> + {% else %} + <div class="row-fluid" id="empty-state-{{table_name}}"> + <div class="alert alert-info">Sorry - no files found</div> + </div> + {%endif%} + </div> + </div> + </div> + </div> +</div> + +<script> + +function _ajax_update(file, all, dir){ + function getCookie(name) { + var cookieValue = null; + if (document.cookie && document.cookie !== '') { + var cookies = document.cookie.split(';'); + for (var i = 0; i < cookies.length; i++) { + var cookie = jQuery.trim(cookies[i]); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) === (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; + } + var csrftoken = getCookie('csrftoken'); + + function csrfSafeMethod(method) { + // these HTTP methods do not require CSRF protection + return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method)); + } + $.ajaxSetup({ + beforeSend: function (xhr, settings) { + if (!csrfSafeMethod(settings.type) && !this.crossDomain) { + xhr.setRequestHeader("X-CSRFToken", csrftoken); + } + } + }); + + $.ajax({ + url:'/toastergui/cmdline/', + type: "POST", + data: {file: file, all: all, dir: dir}, + success:function(data){ + window.location = '/toastergui/builds/' + }, + complete:function(data){ + }, + error:function (xhr, textStatus, thrownError){ + console.log('fail'); + } + }); +} + +$('#import_all').on('click', function(){ + _ajax_update("{{files | safe}}", true, "{{dir | safe}}"); +}); + + +$('#import_page').hide(); + +$(function () { + $('[data-toggle="tooltip"]').tooltip() +}) + + +$("#id_eventlog_file").change(function(){ + console.log($('#file_import')) + $('#file_import').prop("disabled", false); + $('#file_import').addClass('btn-primary') + $('#file_import').removeClass('btn-default') +}) + +</script> + +{% endblock %} diff --git a/lib/toaster/toastergui/urls.py b/lib/toaster/toastergui/urls.py index bc3b0c79..62629494 100644 --- a/lib/toaster/toastergui/urls.py +++ b/lib/toaster/toastergui/urls.py @@ -95,6 +95,7 @@ urlpatterns = [ # project URLs url(r'^newproject/$', views.newproject, name='newproject'), + url(r'^cmdline/$', views.CommandLineBuilds.as_view(), name='cmdlines'), url(r'^projects/$', tables.ProjectsTable.as_view(template_name="projects-toastertable.html"), name='all-projects'), diff --git a/lib/toaster/toastergui/views.py b/lib/toaster/toastergui/views.py index cc8517ba..a75fba47 100644 --- a/lib/toaster/toastergui/views.py +++ b/lib/toaster/toastergui/views.py @@ -6,24 +6,36 @@ # SPDX-License-Identifier: GPL-2.0-only # +import ast import re +import pickle +import codecs +import subprocess + +import bb.cooker +from bb.ui import toasterui from django.db.models import F, Q, Sum from django.db import IntegrityError -from django.shortcuts import render, redirect, get_object_or_404 +from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect from django.utils.http import urlencode from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe from orm.models import LogMessage, Variable, Package_Dependency, Package from orm.models import Task_Dependency, Package_File from orm.models import Target_Installed_Package, Target_File from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File -from orm.models import BitbakeVersion, CustomImageRecipe +from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports from django.urls import reverse, resolve +from django.contrib import messages + from django.core.exceptions import ObjectDoesNotExist +from django.core.files.storage import FileSystemStorage +from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.http import HttpResponseNotFound, JsonResponse from django.utils import timezone +from django.views.generic import TemplateView from datetime import timedelta, datetime from toastergui.templatetags.projecttags import json as jsonfilter from decimal import Decimal @@ -32,6 +44,10 @@ import os from os.path import dirname import mimetypes +from toastergui.forms import LoadFileForm + +from collections import namedtuple + import logging from toastermain.logs import log_view_mixin @@ -41,6 +57,7 @@ logger = logging.getLogger("toaster") # Project creation and managed build enable project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) +import_page = False class MimeTypeFinder(object): # setting this to False enables additional non-standard mimetypes @@ -1940,3 +1957,170 @@ if True: except (ObjectDoesNotExist, IOError): return toaster_render(request, "unavailable_artifact.html") +class EventPlayer: + """Emulate a connection to a bitbake server.""" + + def __init__(self, eventfile, variables): + self.eventfile = eventfile + self.variables = variables + self.eventmask = [] + + def waitEvent(self, _timeout): + """Read event from the file.""" + line = self.eventfile.readline().strip() + if not line: + return + try: + event_str = json.loads(line)['vars'].encode('utf-8') + event = pickle.loads(codecs.decode(event_str, 'base64')) + event_name = "%s.%s" % (event.__module__, event.__class__.__name__) + if event_name not in self.eventmask: + return + return event + except ValueError as err: + print("Failed loading ", line) + raise err + + def runCommand(self, command_line): + """Emulate running a command on the server.""" + name = command_line[0] + + if name == "getVariable": + var_name = command_line[1] + variable = self.variables.get(var_name) + if variable: + return variable['v'], None + return None, "Missing variable %s" % var_name + + elif name == "getAllKeysWithFlags": + dump = {} + flaglist = command_line[1] + for key, val in self.variables.items(): + try: + if not key.startswith("__"): + dump[key] = { + 'v': val['v'], + 'history' : val['history'], + } + for flag in flaglist: + dump[key][flag] = val[flag] + except Exception as err: + print(err) + return (dump, None) + + elif name == 'setEventMask': + self.eventmask = command_line[-1] + return True, None + + else: + raise Exception("Command %s not implemented" % command_line[0]) + + def getEventHandle(self): + """ + This method is called by toasterui. + The return value is passed to self.runCommand but not used there. + """ + pass + + +class CommandLineBuilds(TemplateView): + model = EventLogsImports + template_name = 'command_line_builds.html' + + def get_context_data(self, **kwargs): + context = super(CommandLineBuilds, self).get_context_data(**kwargs) + #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf + eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value']) + if eventlog: + logs_dir = os.path.dirname(eventlog.decode().strip('\n')) + files = os.listdir(logs_dir) + imported_files = EventLogsImports.objects.all() + files_list = [] + + # Filter files that end with ".json" + event_files = [file for file in files if file.endswith(".json")] + + #build dict for template using db data + for event_file in event_files: + if imported_files.filter(name=event_file): + files_list.append({ + 'name': event_file, + 'imported': True, + 'build_id': imported_files.filter(name=event_file)[0].build_id + }) + else: + files_list.append({ + 'name': event_file, + 'imported': False, + 'build_id': None + }) + context['import_all'] = True + + context['files'] = files_list + context['dir'] = logs_dir + else: + context['files'] = [] + context['dir'] = '' + + context['form'] = LoadFileForm() + context['project_enable'] = project_enable + return context + + def post(self, request, **kwargs): + logs_dir = request.POST.get('dir') + all_files = request.POST.get('all') + + imported_files = EventLogsImports.objects.all() + try: + if all_files == 'true': + files = ast.literal_eval(request.POST.get('file')) + for file in files: + if imported_files.filter(name=file.get('name')).exists(): + imported_files.filter(name=file.get('name'))[0].imported = True + else: + with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile: + # load variables from the first line + variables = json.loads(eventfile.readline().strip())['allvariables'] + + params = namedtuple('ConfigParams', ['observe_only'])(True) + player = EventPlayer(eventfile, variables) + + toasterui.main(player, player, params) + event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True) + event_log_import.build_id = Build.objects.last().id + event_log_import.save() + else: + if self.request.FILES.get('eventlog_file'): + file = self.request.FILES['eventlog_file'] + else: + file = request.POST.get('file') + + if imported_files.filter(name=file).exists(): + imported_files.filter(name=file)[0].imported = True + else: + if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile): + variables = json.loads(file.readline().strip())['allvariables'] + params = namedtuple('ConfigParams', ['observe_only'])(True) + player = EventPlayer(file, variables) + if not os.path.exists('{}/{}'.format(logs_dir, file.name)): + fs = FileSystemStorage(location=logs_dir) + fs.save(file.name, file) + toasterui.main(player, player, params) + else: + with open("{}/{}".format(logs_dir, file)) as eventfile: + # load variables from the first line + variables = json.loads(eventfile.readline().strip())['allvariables'] + params = namedtuple('ConfigParams', ['observe_only'])(True) + player = EventPlayer(eventfile, variables) + toasterui.main(player, player, params) + event_log_import = EventLogsImports.objects.create(name=file, imported=True) + event_log_import.build_id = Build.objects.last().id + event_log_import.save() + except json.decoder.JSONDecodeError: + messages.add_message( + self.request, + messages.SUCCESS, + "The file content is not in the correct format. Update file content or upload a different file." + ) + return HttpResponseRedirect("/toastergui/cmdline/") + return HttpResponseRedirect('/toastergui/builds/') -- 2.34.1
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#6059): https://lists.yoctoproject.org/g/toaster/message/6059 Mute This Topic: https://lists.yoctoproject.org/mt/103023934/21656 Group Owner: [email protected] Unsubscribe: https://lists.yoctoproject.org/g/toaster/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
