Andrew Bogott has submitted this change and it was merged. Change subject: Updated dc-migrate a bit. ......................................................................
Updated dc-migrate a bit. I added a few naps to avoid race conditions, and added a salt line to enable clearing of the puppet master certs post-migration. Change-Id: I06179bb5b0714253c25a07b2a7f73ccb0aede196 --- D files/openstack/folsom/virtscripts/dc-migrate M files/openstack/havana/virtscripts/dc-migrate M manifests/role/nova.pp 3 files changed, 11 insertions(+), 78 deletions(-) Approvals: Andrew Bogott: Looks good to me, approved jenkins-bot: Verified diff --git a/files/openstack/folsom/virtscripts/dc-migrate b/files/openstack/folsom/virtscripts/dc-migrate deleted file mode 100755 index 53d9fda..0000000 --- a/files/openstack/folsom/virtscripts/dc-migrate +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -##################################################################### -### THIS FILE IS MANAGED BY PUPPET -### puppet:///files/openstack/folsom/virtscripts/dc-migrate -##################################################################### -#set -x -ssh-add -l > /dev/null -if [ "$?" != "0" ] -then - echo "You must have a forwarded agent to use this script" - exit 1 -fi -INSTANCENAME=$1 -PROJECT=$2 -FROMREGION='pmtpa' -TOREGION='eqiad' -FROMAUTH="http://virt0.wikimedia.org:35357/v2.0"; -TOAUTH="http://virt1000.wikimedia.org:35357/v2.0"; -FROMPUPPETMASTER="virt0.wikimedia.org" -TOPUPPETMASTER="virt1000.wikimedia.org" -TOINSTANCE=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep ' id ' | awk '{ print $4 }'` -if [ -n "$TOINSTANCE" ]; then - echo "Name conflict! ${INSTANCENAME} already exists in ${TOREGION} with ID \"${TOINSTANCE}\"" - exit 1 -fi -echo "(That error was a good thing.)" -echo "" -FROMIMAGE=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${FROMAUTH} nova --os-region ${FROMREGION} show ${INSTANCENAME} | grep ' image ' | awk '{gsub ( "[()]","" ) ; print $(NF-1) }'` -FROMFLAVOR=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${FROMAUTH} nova --os-region ${FROMREGION} show ${INSTANCENAME} | grep ' flavor ' | awk '{gsub ( "[()]","" ) ; print $5 }'` -FROMSECURITYRAW=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${FROMAUTH} nova --os-region ${FROMREGION} show ${INSTANCENAME} | grep ' security_groups ' | awk -F "|" '{ print $3 }'` -FROMSECURITY=`python -c "print ','.join([o['name'] for o in ${FROMSECURITYRAW}])"` -FROMHOST=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${FROMAUTH} nova --os-region ${FROMREGION} show ${INSTANCENAME} | grep 'OS-EXT-SRV-ATTR:host' | awk '{ print $4 }'` -FROMINSTANCENAME=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${FROMAUTH} nova --os-region ${FROMREGION} show ${INSTANCENAME} | grep 'OS-EXT-SRV-ATTR:instance_name' | awk '{ print $4 }'` -FROMINSTANCE=$FROMINSTANCENAME -echo "Creating initial image in $TOREGION" -cd /srv/org/wikimedia/controller/wikis/w && php ./extensions/OpenStackManager/maintenance/createInstance.php --image ${FROMIMAGE} --flavor ${FROMFLAVOR} --instance ${INSTANCENAME} --project ${PROJECT} --region ${TOREGION} --securitygroups ${FROMSECURITY} -echo "The new instance is building. Waiting for it to be assigned a host..." -TOHOST='' -while [ -z $TOHOST ]; do - sleep 10 - TOHOST=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep 'OS-EXT-SRV-ATTR:host' | awk '{ print $4 }'` -done -TOINSTANCENAME=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep 'OS-EXT-SRV-ATTR:instance_name' | awk '{ print $4 }'` -TOINSTANCE=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep ' id ' | awk '{ print $4 }'` -echo "" -echo "Migrating $FROMINSTANCE from $FROMHOST to $TOINSTANCE on $TOHOST" -echo "" -echo "Destroying new instance on $TOHOST" -echo "" -ssh $TOHOST.$TOREGION.wmnet "virsh destroy ${TOINSTANCE}" -echo "Clearing resolv.conf in instance on $FROMHOST" -echo "" -salt ${FROMINSTANCE}.${FROMREGION}.wmflabs cmd.run "rm -f /etc/resolv.conf" -echo "Destroying instance on $FROMHOST" -echo "" -ssh $FROMHOST.$FROMREGION.wmnet "virsh destroy ${FROMINSTANCE}" -echo "" -sleep 30 -echo "" -echo "Rsyncing instance from $FROMHOST to $TOHOST" -echo "" -# This one is right, but currently impossible due to filtering -#ssh -A $FROMHOST.$FROMREGION.wmnet "rsync -S -av /var/lib/nova/instances/${FROMINSTANCE} ${TOHOST}.${TOREGION}.wmnet:/var/lib/nova/instances/${TOINSTANCE}" -# This is dumb but will work -rsync -S -av ${FROMHOST}.${FROMREGION}.wmnet:/var/lib/nova/instances/${FROMINSTANCE} /tmp/ -rsync -S -av /tmp/${FROMINSTANCE}/* ${TOHOST}.${TOREGION}.wmnet:/var/lib/nova/instances/${TOINSTANCE} -rm -r /tmp/${FROMINSTANCE} -echo "" -echo "" -echo "" -echo "Clearing stale puppet cert (if any)" -puppet cert clean ${TOINSTANCENAME}.${TOREGION}.wmflabs -echo "" -echo "Rebooting the instance" -echo "" -OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} reboot ${INSTANCENAME} diff --git a/files/openstack/havana/virtscripts/dc-migrate b/files/openstack/havana/virtscripts/dc-migrate index 540d360..460d57f 100755 --- a/files/openstack/havana/virtscripts/dc-migrate +++ b/files/openstack/havana/virtscripts/dc-migrate @@ -42,6 +42,7 @@ done TOINSTANCENAME=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep 'OS-EXT-SRV-ATTR:instance_name' | awk '{ print $4 }'` TOINSTANCE=`OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} show ${INSTANCENAME} | grep ' id ' | awk '{ print $4 }'` +sleep 2 echo "" echo "Migrating $FROMINSTANCE from $FROMHOST to $TOINSTANCE on $TOHOST" echo "" @@ -51,11 +52,19 @@ echo "Clearing resolv.conf in instance on $FROMHOST" echo "" salt ${FROMINSTANCE}.${FROMREGION}.wmflabs cmd.run "rm -f /etc/resolv.conf" +salt ${FROMINSTANCE}.${FROMREGION}.wmflabs cmd.run "rm -f /etc/resolvconf/resolv.conf.d/original" +echo "" +echo "Enabling puppet master cert refresh" +salt ${FROMINSTANCE}.${FROMREGION}.wmflabs cmd.run "touch /root/allowcertdeletion" +echo "" +sleep 1 echo "Destroying instance on $FROMHOST" echo "" +OS_TENANT_NAME=${PROJECT} OS_AUTH_URL=${TOAUTH} nova --os-region ${TOREGION} stop ${INSTANCENAME} +sleep 5 ssh $FROMHOST.$FROMREGION.wmnet "virsh destroy ${FROMINSTANCE}" echo "" -sleep 30 +sleep 10 echo "" echo "Rsyncing instance from $FROMHOST to $TOHOST" echo "" diff --git a/manifests/role/nova.pp b/manifests/role/nova.pp index dd8c424..336fe42 100644 --- a/manifests/role/nova.pp +++ b/manifests/role/nova.pp @@ -318,7 +318,7 @@ # Script to migrate instance from one dc to another (specifically, pmtpa to eqiad) file { "/root/dc-migrate": ensure => present, - source => "puppet:///files/openstack/${openstack_version}/virtscripts/dc-migrate", + source => "puppet:///files/openstack/havana/virtscripts/dc-migrate", mode => 0755, owner => root, group => root, -- To view, visit https://gerrit.wikimedia.org/r/115606 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I06179bb5b0714253c25a07b2a7f73ccb0aede196 Gerrit-PatchSet: 3 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Andrew Bogott <[email protected]> Gerrit-Reviewer: Andrew Bogott <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
