On 2015-12-18, Holger Levsen wrote:
> Hi Vagrant,
>
> On Samstag, 19. Dezember 2015, Vagrant Cascadian wrote:
>> Patch below! No idea if it works, given that I don't have a spare
>> jenkins.debian.net or build network to test on, but hopefully it
>> demonstrates the idea, and is mostly there.
>
> as discussed on irc: thanks for your work on this! I like the idea, I very 
> much like the idea of developing this as a 16th (or N+1th) build job and in 
> general I'm willing to give it a try.

I didn't spend any time really figuring out which nodes to add to the
example 16th build job, so that might need some adjusting.


> Just please address those "FIXME"s first, and second the concerns I've
> raised so far on IRC, and third, future concerns ;-)

Ok, updated patch should address those concerns and a few obvious bugfixes:

- Split load estimating into it's own script, and add support for
available memory.

- Only set NODE* and PORT* from build_rebuild when called from
build_rebuild function.

- .debian.net on NODE1 and NODE2 in build_rebuild function.

- Call timeout so that the ssh processes don't take too long to complete.

diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 338c207..28dd8ed 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -688,9 +688,43 @@ check_buildinfo() {
        rm -f $TMPFILE1 $TMPFILE2
 }
 
+select_least_loaded_node() {
+    local pool_nodes
+    local node
+    local port
+    local load
+    local best_load
+    local selected
+    # default to the first node
+    selected="$1"
+    pool_nodes="$@"
+    if [ "$selected" = "$pool_nodes" ]; then
+       echo $selected
+       return 0
+    fi
+    load = 0
+    best_load = 0
+    for this_node in $pool_nodes ; do
+       node=$(echo $this_node | cut -d : -f 1)
+       port=$(echo $this_node | cut -d : -f 2)
+       load=$(timeout 10 ssh $node -p $port 
/srv/jenkins/bin/reproducible_build_node_load.sh)
+       if [ "$load" -gt "$best_load" ]; then
+           selected="$this_node"
+           best_load="$load"
+       fi
+    done
+    echo $selected
+}
+
 build_rebuild() {
        FTBFS=1
        mkdir b1 b2
+       local selected_node
+       selected_node=$(select_least_loaded_node $NODE1_POOL)
+       NODE1=$(echo $selected_node | cut -d : -f 1).debian.net
+       PORT1=$(echo $selected_node | cut -d : -f 2)
+       # if no port is given, assume 22
+       if [ "$NODE1" = "${PORT1}.debian.net" ] ; then PORT1=22 ; fi
        remote_build 1 $NODE1 $PORT1
        if [ ! -f b1/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] && [ -f 
b1/${SRCPACKAGE}_*_${ARCH}.changes ] ; then
                        echo "Version mismatch between main node 
(${SRCPACKAGE}_${EVERSION}_${ARCH}.dsc expected) and first build node ($(ls 
b1/*dsc)) for $SUITE/$ARCH, aborting. Please upgrade the schroots..." | tee -a 
${RBUILDLOG}
@@ -700,6 +734,11 @@ build_rebuild() {
                        exit 0
        elif [ -f b1/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then
                # the first build did not FTBFS, try rebuild it.
+               selected_node=$(select_least_loaded_node $NODE2_POOL)
+               NODE2=$(echo $selected_node | cut -d : -f 1).debian.net
+               PORT2=$(echo $selected_node | cut -d : -f 2)
+               # if no port is given, assume 22
+               if [ "$NODE2" = "${PORT2}.debian.net" ] ; then PORT2=22 ; fi
                remote_build 2 $NODE2 $PORT2
                if [ -f b2/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then
                        # both builds were fine, i.e., they did not FTBFS.
@@ -750,17 +789,12 @@ elif [ "$1" = "1" ] || [ "$1" = "2" ] ; then
        exit 0
 elif [ "$2" != "" ] ; then
        MODE="master"
-       NODE1="$(echo $1 | cut -d ':' -f1).debian.net"
-       NODE2="$(echo $2 | cut -d ':' -f1).debian.net"
-       PORT1="$(echo $1 | cut -d ':' -f2)"
-       PORT2="$(echo $2 | cut -d ':' -f2)"
-       # if no port is given, assume 22
-       if [ "$NODE1" = "${PORT1}.debian.net" ] ; then PORT1=22 ; fi
-       if [ "$NODE2" = "${PORT2}.debian.net" ] ; then PORT2=22 ; fi
+       NODE1_POOL="$1"
+       NODE2_POOL="$2"
        # overwrite ARCH for remote builds
        for i in $ARCHS ; do
                # try to match ARCH in nodenames
-               if [[ "$NODE1" =~ .*-$i.* ]] ; then
+               if [[ "$NODE1_POOL" =~ .*-$i.* ]] ; then
                        ARCH=$i
                fi
        done
diff --git a/bin/reproducible_node_load.sh b/bin/reproducible_node_load.sh
new file mode 100755
index 0000000..36bce84
--- /dev/null
+++ b/bin/reproducible_node_load.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# produce an estimated load based on the number of CPUs and RAM, minus
+# the load average.
+
+cpus=$(grep ^processor /proc/cpuinfo | wc -l)
+memory_available=$(awk '/^MemAvailable:/{print $2}' /proc/meminfo)
+loadavg=$(awk '{print $1}' /proc/loadavg)
+fudge=100
+
+# Calculation is basically, memory + cpus - loadavg, with some fudge
+# factor to try to ensure it's only a positive number.
+echo "$memory_available + ($cpus - $loadavg + $fudge) * 1000000" | \
+    bc | \
+    cut -d . -f 1
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 97df606..5963206 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -481,7 +481,8 @@
                 - '13': { my_node1: 'cbxi4pro0-armhf-rb:2226', my_node2: 
'rpi2b-armhf-rb:2230'     }
                 - '14': { my_node1: 'rpi2b-armhf-rb:2230',     my_node2: 
'wbq0-armhf-rb:2225'      }
                 - '15': { my_node1: 'wbd0-armhf-rb:2223',      my_node2: 
'hb0-armhf-rb:2224'       }
-            my_shell: '/srv/jenkins/bin/reproducible_build.sh {my_node1} 
{my_node2}'
+                - '16': { my_node1: 'wbd0-armhf-rb:2223 wbq0-armhf-r:2225', 
my_node2: 'bpi0-armhf-rb:2222 odxu4-armhf-rb:2229' }
+            my_shell: '/srv/jenkins/bin/reproducible_build.sh "{my_node1}" 
"{my_node2}"'
             my_timed: '* * * * *'
             my_hname: ''
 

> btw, there's jenkins-test-vm.debian.net now too, but currently it's mostly 
> only suited for testing job configurations…

Ah well, someday... :)


live well,
  vagrant

Attachment: signature.asc
Description: PGP signature

_______________________________________________
Reproducible-builds mailing list
Reproducible-builds@lists.alioth.debian.org
http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/reproducible-builds

Reply via email to