I wrote this tool to do automatic updating of layers managed by the
setup-layers script. This
addresses some of the functionality I discussed in my OSSNA 2023 talk
<https://www.youtube.com/watch?v=MiTwL9xZdW4>.
A detailed explanation of what the following script does is found in the
comments, but in
a nutshell:
* If the --sync-upstream argument is provided, any layers that have remotes
called
"origin" and "upstream" have upstream changes on their configured branch
applied
to the origin remote (and local clone).
* Updates rev and description for all layers managed by setup-layers after
pulling
the latest changes on the configured branch.
* Leaves the setup-layers.json file in an updated state so git diff will
show changes.
Changes can then be reviewed and managed based on local project policy.
I am looking for comments and feedback before I send this as a proper patch.
In particular:
* Are there any problems with this general approach?
* Should this be managed in OE-Core in the same way setup-layers is?
* I made some stylistic changes (e.g. main function, --dry-run, etc) that I
plan to also patch into
setup-layers if this is accepted and there are no objections.
---
#!/usr/bin/env python3
#
# Copyright OpenEmbedded Contributors
#
# SPDX-License-Identifier: MIT
#
# This script is idempotent and assumes that the layers it is updating have
# been managed by setup-layers according to the JSON formatted layers data.
#
# For each layer repository in the JSON formatted layers data file:
#
# * If remotes named upstream and origin exist and the --sync-upstream
argument
# has been provided, merge upstream changes on the configured branch to the
# origin remote.
# * Pull the latest changes on the configured branch from origin.
# * Update the JSON layers data to reflect the latest layer repo HEAD.
# * Leave layer repository in an updated state with the configured branch
# checked out.
#
# After reviewing the changes in the JSON layers data file, they can either
be
# committed or reverted. If they are reverted, it will be necessary to
re-run
# setup-layers to ensure your layer repositories reflect the intent
captured in
# the configuration. For the sake of completeness, you may also wish to
re-run
# setup-layers even if you commit the configuration changes, as this script
# leaves the local layer repo clones checked out to the configured branch
rather
# than the configured rev. The HEAD of the configured branch and the
configured
# rev are identical at the completion of this script, but the latter will be
# more consistent with expectations established by the setup-layers script.
#
# The term "setup-layers" refers to the script found at
# openembedded-core/scripts/oe-setup-layers. Pass the --help argument to
this
# script for the most up to date usage guidance.
import argparse
import json
import os
import subprocess
def _do_git_command(repodir, command):
print("Running '{}' in {}".format(command, repodir))
if not args.dry_run:
subprocess.check_output(command, shell=True, cwd=repodir)
def _do_git_checkout_branch(repodir, remote, branch):
_do_git_command(repodir, 'git checkout -B {0} --track {1}/{0}
--quiet'.format(branch, remote))
def _do_git_pull(repodir, remote, branch):
_do_git_fetch(repodir, remote)
_do_git_merge(repodir, remote, branch)
def _do_git_fetch(repodir, remote):
_do_git_command(repodir, 'git fetch {} --prune --prune-tags
--quiet'.format(remote))
def _do_git_merge(repodir, remote, branch):
_do_git_command(repodir, 'git merge {}/{} --ff-only
--quiet'.format(remote, branch))
def _do_git_force_push(repodir, remote, branch):
_do_git_command(repodir, 'git push {} heads/{} --force
--quiet'.format(remote, branch))
def _get_git_rev(repodir):
rev = subprocess.check_output("git rev-parse HEAD", shell=True,
cwd=repodir, stderr=subprocess.DEVNULL)
return rev.strip().decode("utf-8")
def _get_git_desc(repodir):
desc = subprocess.check_output("git describe HEAD --always",
shell=True, cwd=repodir, stderr=subprocess.DEVNULL)
return desc.strip().decode("utf-8")
def _do_sync_upstream(repodir, repo_name, repo_data):
if not args.sync_upstream:
return
repo_remote = repo_data['git-remote']
branch = repo_remote['branch']
remotes = repo_remote['remotes']
print("Attempting to sync layer repo {} with
upstream.".format(repo_name))
if 'origin' not in remotes or 'upstream' not in remotes:
print("Origin and/or upstream remotes not found on
{}".format(repo_name))
return
_do_git_checkout_branch(repodir, 'origin', branch)
_do_git_pull(repodir, 'origin', branch)
_do_git_fetch(repodir, 'upstream')
_do_git_merge(repodir, 'upstream', branch)
_do_git_force_push(repodir, 'origin', branch)
def _do_update_layer_json(repodir, repo_name, layer_data):
print("Updating layer JSON for {} repo.".format(repo_name))
branch = layer_data['sources'][repo_name]['git-remote']['branch']
_do_git_checkout_branch(repodir, 'origin', branch)
_do_git_pull(repodir, 'origin', branch)
layer_data['sources'][repo_name]['git-remote']['rev'] =
_get_git_rev(repodir)
layer_data['sources'][repo_name]['git-remote']['describe'] =
_get_git_desc(repodir)
def _do_update_layers(layer_data):
print("Updating layers...")
repos = layer_data['sources']
for repo_name in repos:
repo_data = repos[repo_name]
repodir = os.path.abspath(os.path.join(args.destdir,
repo_data['path']))
if 'contains_this_file' in repo_data.keys():
continue
_do_sync_upstream(repodir, repo_name, repo_data)
_do_update_layer_json(repodir, repo_name, layer_data)
def _do_save_layer_config(layer_data):
print("Saving updated layer config.")
if args.dry_run:
return
with open(args.jsondata, 'w') as f:
json.dump(layer_data, f, ensure_ascii=False, indent=4)
print("\nReview {} for layer changes.".format(args.jsondata))
def _parse_args():
global args
try:
defaultdest = os.path.dirname(subprocess.check_output('git
rev-parse --show-toplevel', universal_newlines=True, shell=True,
cwd=os.path.dirname(__file__)))
except subprocess.CalledProcessError as e:
defaultdest = os.path.abspath(".")
parser = argparse.ArgumentParser(description="Updates layer setup JSON
in addition to layer repos with upstream remotes.")
parser.add_argument('--destdir', default=defaultdest, help='Where to
find the layers (default is
{defaultdest}).'.format(defaultdest=defaultdest))
parser.add_argument('--jsondata', default=__file__+".json", help='File
containing the layer data in json format (default is
{defaultjson}).'.format(defaultjson=__file__+".json"))
parser.add_argument('--sync-upstream', action='store_true', help="For
repositories with origin and upstream remotes, merge any upstream changes
to the configured branch.")
parser.add_argument('--dry-run', action='store_true', help="Performs a
dry run, but does not make any actual changes.")
args = parser.parse_args()
def _parse_layer_config():
with open(args.jsondata) as f:
layer_data = json.load(f)
supported_versions = ["1.0"]
if layer_data["version"] not in supported_versions:
raise Exception("File {} has version {}, which is not in supported
versions: {}".format(args.jsondata, layer_data["version"],
supported_versions))
return layer_data
def main():
_parse_args()
layer_data = _parse_layer_config()
_do_update_layers(layer_data)
_do_save_layer_config(layer_data)
if __name__=="__main__":
main()
--
"Perfection must be reached by degrees; she requires the slow hand of
time." - Voltaire
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#184211):
https://lists.openembedded.org/g/openembedded-core/message/184211
Mute This Topic: https://lists.openembedded.org/mt/100104869/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
