laforge has submitted this change. (
https://gerrit.osmocom.org/c/osmo-ci/+/42095?usp=email )
Change subject: scripts/kernel/linux-shallow-clone: new script
......................................................................
scripts/kernel/linux-shallow-clone: new script
Add a script that maintains a linux repository in one place on jenkins
nodes, so we need less git clones from git.kernel.org and less disk
space. All jobs that need a kernel tree can now clone the relevant
branch directly from the jenkins node.
Follow up patches will add a jenkins job that runs the script daily and
adjust the existing jobs to make use of this instead of doing their own
clones.
Currently this script produces a 396M bare git repository.
Related: OS#6938
Change-Id: Id3aadb46813047ecac3b80868192809b272dba0e
---
A scripts/kernel/linux-shallow-clone.sh
1 file changed, 97 insertions(+), 0 deletions(-)
Approvals:
laforge: Looks good to me, approved
Jenkins Builder: Verified
pespin: Looks good to me, but someone else must approve
diff --git a/scripts/kernel/linux-shallow-clone.sh
b/scripts/kernel/linux-shallow-clone.sh
new file mode 100755
index 0000000..3aa9267
--- /dev/null
+++ b/scripts/kernel/linux-shallow-clone.sh
@@ -0,0 +1,97 @@
+#!/bin/sh -ex
+# Create a bare git repository and add/update shallow clones of linux.git
+# branches that are relevant for our CI jobs. Jobs can then quickly clone a
+# branch from this git repository and discard it afterwards. This saves disk
+# space on our jenkins nodes, and keeps the traffic to git.kernel.org minimal.
+
+set_dest() {
+ if [ -n "$1" ]; then
+ DEST="$1"
+ else
+ DEST=/tmp/linux-shallow
+ fi
+
+ # Check if an existing repository is from last month, and if that is
+ # the case then start with a new repository. This is done because the
+ # repository size increases with each fetch: we start with a shallow
+ # clone but can't truncate the history after follow-up fetches.
+ if [ -e "$DEST/.month" ] && [ "$(cat "$DEST/.month")" != "$(date +%m)"
]; then
+ DEST_OLD="$DEST"
+ DEST="$DEST-new"
+ fi
+}
+
+init() {
+ if [ -d "$DEST" ]; then
+ return
+ fi
+
+ mkdir -p "$DEST"
+ cd "$DEST"
+ git init . --bare
+
+ # Garbage collect in foreground
+ git config gc.autoDetach false
+
+ git remote add torvalds
"https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git";
+ git remote add stable
"https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git";
+ git remote add net-next
"https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git";
+
+ echo "$(date +%m)" > "$DEST/.month"
+}
+
+fetch_branch() {
+ local remote="$1"
+ local branch="$2"
+ local head
+
+ git fetch --no-tags --depth=1 "$remote" "$branch"
+ head="$(git log -n1 --pretty=format:"%H" "$remote"/"$branch")"
+ git branch --force "$remote-$branch" "$head"
+
+ # Pretty print the current commit for the log
+ git -c color.ui=always log -1 --oneline "$remote-$branch"
+}
+
+fetch_branches() {
+ fetch_branch torvalds "master"
+
+ fetch_branch stable "linux-rolling-stable"
+ fetch_branch stable "linux-4.19.y"
+ fetch_branch stable "linux-5.10.y"
+ fetch_branch stable "linux-6.1.y"
+ fetch_branch stable "linux-6.12.y"
+
+ fetch_branch net-next "main"
+}
+
+collect_garbage() {
+ git gc --no-cruft --prune=now
+}
+
+# Replace last month's repository if needed, see the comment in set_dest
+replace_old() {
+ if [ -z "$DEST_OLD" ]; then
+ return
+ fi
+
+ mv "$DEST_OLD" "$DEST"-old
+ mv "$DEST" "$DEST_OLD"
+ rm -rf "$DEST"-old
+
+ DEST="$DEST_OLD"
+ DEST_OLD=""
+ cd "$DEST"
+}
+
+show_size() {
+ du -h -s "$DEST"
+}
+
+set_dest "$@"
+init
+cd "$DEST"
+fetch_branches
+collect_garbage
+replace_old
+show_size
--
To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/42095?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.osmocom.org/settings?usp=email
Gerrit-MessageType: merged
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Change-Id: Id3aadb46813047ecac3b80868192809b272dba0e
Gerrit-Change-Number: 42095
Gerrit-PatchSet: 1
Gerrit-Owner: osmith <[email protected]>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: fixeria <[email protected]>
Gerrit-Reviewer: laforge <[email protected]>
Gerrit-Reviewer: pespin <[email protected]>
