Hello community, here is the log from the commit of package WALinuxAgent for openSUSE:Factory checked in at 2015-04-16 14:12:31 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/WALinuxAgent (Old) and /work/SRC/openSUSE:Factory/.WALinuxAgent.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "WALinuxAgent" Changes: -------- --- /work/SRC/openSUSE:Factory/WALinuxAgent/WALinuxAgent.changes 2015-01-23 15:46:12.000000000 +0100 +++ /work/SRC/openSUSE:Factory/.WALinuxAgent.new/WALinuxAgent.changes 2015-04-16 14:12:32.000000000 +0200 @@ -1,0 +2,7 @@ +Wed Mar 25 07:13:39 UTC 2015 - rjsch...@suse.com + +- Update to version 2.0.12 (bnc#924135,bnc#924137i,bnc#919244) + + Add support for page blob status report + + Restart the service on upgrade + +------------------------------------------------------------------- Old: ---- WALinuxAgent-2.0.11.tar.gz New: ---- WALinuxAgent-2.0.12.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ WALinuxAgent.spec ++++++ --- /var/tmp/diff_new_pack.MKMU1p/_old 2015-04-16 14:12:32.000000000 +0200 +++ /var/tmp/diff_new_pack.MKMU1p/_new 2015-04-16 14:12:32.000000000 +0200 @@ -1,7 +1,7 @@ # # spec file for package WALinuxAgent # -# Copyright (c) 2014 SUSE LINUX Products GmbH, Nuernberg, Germany. +# Copyright (c) 2015 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -20,7 +20,7 @@ Summary: The Windows Azure Linux Agent License: Apache-2.0 Group: System/Daemons -Version: 2.0.11 +Version: 2.0.12 Release: 0 Url: https://github.com/Azure/WALinuxAgent Source0: https://github.com/Azure/%{name}/archive/%{name}-%{version}.tar.gz @@ -128,6 +128,7 @@ %endif %postun +%restart_on_update waagent %if 0%{?suse_version} > 1140 %service_del_postun waagent.service %else ++++++ WALinuxAgent-2.0.11.tar.gz -> WALinuxAgent-2.0.12.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/.gitignore new/WALinuxAgent-WALinuxAgent-2.0.12/.gitignore --- old/WALinuxAgent-WALinuxAgent-2.0.11/.gitignore 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/.gitignore 2015-03-15 15:16:08.000000000 +0100 @@ -1,3 +1,4 @@ waagentc *.py[cod]* +tests/status_blob_url.py diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/Changelog new/WALinuxAgent-WALinuxAgent-2.0.12/Changelog --- old/WALinuxAgent-WALinuxAgent-2.0.11/Changelog 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/Changelog 2015-03-15 15:16:08.000000000 +0100 @@ -1,5 +1,8 @@ WALinuxAgent Changelog ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +15 Jan 2015, WALinuxAgent 2.0.12 + . Add support for page blob status report + 11 Dec 2014, WALinuxAgent 2.0.11 . Add support for GPT(Guid Partition Table) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/README new/WALinuxAgent-WALinuxAgent-2.0.12/README --- old/WALinuxAgent-WALinuxAgent-2.0.11/README 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/README 2015-03-15 15:16:08.000000000 +0100 @@ -1,9 +1,9 @@ -Windows Azure Linux Agent README +Microsoft Azure Linux Agent README INTRODUCTION -The Windows Azure Linux Agent (waagent) manages Linux provisioning and VM -interaction with the Windows Azure Fabric Controller. It provides the following +The Microsoft Azure Linux Agent (waagent) manages Linux & FreeBSD provisioning, +and VM interaction with the Azure Fabric Controller. It provides the following functionality for Linux and FreeBSD IaaS deployments: * Image Provisioning @@ -53,12 +53,13 @@ REQUIREMENTS -The following systems have been tested and are known to work with the Windows -Azure Linux Agent. Please note that this list may differ from the official -list of supported systems on the Windows Azure Platform as described here: +The following systems have been tested and are known to work with the Azure +Linux Agent. Please note that this list may differ from the official list +of supported systems on the Windows Azure Platform as described here: http://support.microsoft.com/kb/2805216 Supported Linux Distributions: + * CoreOS * CentOS 6.2+ * Debian 7.0+ * Ubuntu 12.04+ @@ -67,14 +68,14 @@ * Oracle Linux 6.4+ Other Supported Systems: - * FreeBSD 9+ + * FreeBSD 10+ (Azure Linux Agent v2.0.10+) Waagent depends on some system packages in order to function properly: - * Python 2.5+ + * Python 2.6+ * OpenSSL 1.0+ * OpenSSH 5.3+ - * Filesystem utilities: sfdisk, fdisk, mkfs + * Filesystem utilities: sfdisk, fdisk, mkfs, parted * Password tools: chpasswd, sudo * Text processing tools: sed, grep * Network tools: ip-route @@ -87,8 +88,12 @@ files provided (see debian/README and rpm/README). If installing manually, waagent should be copied to /usr/sbin/waagent and -installed by running: /usr/sbin/waagent -install. The waagent log is kept at -/var/log/waagent.log. +installed by running: + + # sudo chmod 755 /usr/sbin/waagent + # sudo /usr/sbin/waagent -install -verbose + +The agent's log file is kept at /var/log/waagent.log. COMMAND LINE OPTIONS @@ -155,7 +160,7 @@ waagent. A sample configuration file is shown below: # -# Windows Azure Linux Agent Configuration +# Azure Linux Agent Configuration # Role.StateConsumer=None @@ -219,7 +224,7 @@ This allows the user to enable or disable the provisioning functionality in the agent. Valid values are "y" or "n". If provisioning is disabled, SSH host and user keys in the image are preserved and any configuration specified in the -Windows Azure provisioning API is ignored. +Azure provisioning API is ignored. Provisioning.DeleteRootPassword: Type: Boolean Default: n diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/get-agent.py new/WALinuxAgent-WALinuxAgent-2.0.12/get-agent.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/get-agent.py 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/get-agent.py 2015-03-15 15:16:08.000000000 +0100 @@ -28,11 +28,10 @@ import re import platform -def upgrade(): +def upgrade(account='Azure', ref='2.0'): #Define variables - account = 'Azure' agentUri = ('https://raw.githubusercontent.com/{0}/' - 'WALinuxAgent/WALinuxAgent-2.0.11/waagent').format(account) + 'WALinuxAgent/{1}/waagent').format(account, ref) distro = platform.linux_distribution() cmd = ['service', 'waagent', 'restart'] agent_file="/usr/sbin/waagent" @@ -64,4 +63,7 @@ job.wait() if __name__ == '__main__': - upgrade() + if len(sys.argv) == 3: + upgrade(sys.argv[1], sys.argv[2]) + else: + upgrade() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/test/part-gpt.py new/WALinuxAgent-WALinuxAgent-2.0.12/test/part-gpt.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/test/part-gpt.py 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/test/part-gpt.py 1970-01-01 01:00:00.000000000 +0100 @@ -1,31 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2014 Microsoft Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Implements parts of RFC 2131, 1541, 1497 and -# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx -# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx -# - -import subprocess - -if __name__ == '__main__': - subprocess.call(['umount', '/mnt/resource']) - subprocess.call(['umount', '/mnt']) - subprocess.call(['parted', '/dev/sdb', 'print']) - subprocess.call(['parted', '/dev/sdb', 'rm', '1']) - subprocess.call(['parted', '/dev/sdb', 'mklabel', 'gpt']) - subprocess.call(['parted', '/dev/sdb', 'mkpart', 'primary', '0%', '50%']) - subprocess.call(['parted', '/dev/sdb', 'mkpart', 'primary', '50%', '100%']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/tests/env.py new/WALinuxAgent-WALinuxAgent-2.0.12/tests/env.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/tests/env.py 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/tests/env.py 2015-03-15 15:16:08.000000000 +0100 @@ -19,5 +19,5 @@ projet_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) waagent = imp.load_source('waagent', os.path.join(projet_root, 'waagent')) -waagent.LoggerInit('/tmp/test.log','/dev/stdout') +waagent.LoggerInit('/dev/stdout', '/dev/null') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/tests/part-gpt.py new/WALinuxAgent-WALinuxAgent-2.0.12/tests/part-gpt.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/tests/part-gpt.py 1970-01-01 01:00:00.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/tests/part-gpt.py 2015-03-15 15:16:08.000000000 +0100 @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +import subprocess + +if __name__ == '__main__': + subprocess.call(['umount', '/mnt/resource']) + subprocess.call(['umount', '/mnt']) + subprocess.call(['parted', '/dev/sdb', 'print']) + subprocess.call(['parted', '/dev/sdb', 'rm', '1']) + subprocess.call(['parted', '/dev/sdb', 'mklabel', 'gpt']) + subprocess.call(['parted', '/dev/sdb', 'mkpart', 'primary', '0%', '50%']) + subprocess.call(['parted', '/dev/sdb', 'mkpart', 'primary', '50%', '100%']) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/tests/test_shared_config.py new/WALinuxAgent-WALinuxAgent-2.0.12/tests/test_shared_config.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/tests/test_shared_config.py 1970-01-01 01:00:00.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/tests/test_shared_config.py 2015-03-15 15:16:08.000000000 +0100 @@ -0,0 +1,85 @@ +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest +from env import waagent + + +class TestSharedConfig(unittest.TestCase): + + def test_parse_shared_config(self): + conf = waagent.SharedConfig().Parse(SharedConfigText) + self.assertNotEquals(None, conf) + self.assertNotEquals(None, conf.RdmaMacAddress) + self.assertNotEquals(None, conf.RdmaIPv4Address) + return conf + + def test_config_rdma(self): + waagent.LoggerInit("/dev/stdout", "/dev/null", verbose=True) + testDev = "/tmp/hvnd_rdma" + waagent.SetFileContents(testDev, "") + conf = self.test_parse_shared_config() + conf.ConfigRdma(dev=testDev) + rdmaConf = waagent.GetFileContents(testDev) + self.assertNotEquals(None, rdmaConf) + self.assertNotEquals("", rdmaConf) + +SharedConfigText="""\ +<?xml version="1.0" encoding="utf-8"?> +<SharedConfig version="1.0.0.0" goalStateIncarnation="1"> + <Deployment name="698f959e434c41cc9d72a2c67c044463" guid="{ba92e945-0302-4030-9710-257c03c07e22}" incarnation="0" isNonCancellableTopologyChangeEnabled="false"> + <Service name="test-rdms" guid="{00000000-0000-0000-0000-000000000000}" /> + <ServiceInstance name="698f959e434c41cc9d72a2c67c044463.0" guid="{6f157bcb-b6ac-4fdd-9789-2ca466220e17}" /> + </Deployment> + <Incarnation number="1" instance="test-rdms" guid="{33d19bb6-f34d-4dfb-966c-2bade1714cc5}" /> + <Role guid="{dad0becc-5d1d-3c55-3285-0136e9933bbe}" name="test-rdms" settleTimeSeconds="0" /> + <LoadBalancerSettings timeoutSeconds="0" waitLoadBalancerProbeCount="8"> + <Probes> + <Probe name="D41D8CD98F00B204E9800998ECF8427E" /> + <Probe name="423A4BBA20CEBE79BA641B20A03ED6F9" /> + </Probes> + </LoadBalancerSettings> + <OutputEndpoints> + <Endpoint name="test-rdms:openInternalEndpoint" type="SFS"> + <Target instance="test-rdms" endpoint="openInternalEndpoint" /> + </Endpoint> + </OutputEndpoints> + <Instances> + <Instance id="test-rdms" address="100.74.58.20" primaryMacAddress="000D3A101ED4" rdmaMacAddress="00155D340044" rdmaIPv4Address="172.16.2.59"> + <FaultDomains randomId="0" updateId="0" updateCount="0" /> + <InputEndpoints> + <Endpoint name="openInternalEndpoint" address="100.74.58.20" protocol="any" isPublic="false" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false"> + <LocalPorts> + <LocalPortSelfManaged /> + </LocalPorts> + </Endpoint> + <Endpoint name="SSH" address="100.74.58.20:22" protocol="tcp" hostName="test-rdmsContractContract" isPublic="true" loadBalancedPublicAddress="104.45.128.35:22" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false"> + <LocalPorts> + <LocalPortRange from="22" to="22" /> + </LocalPorts> + </Endpoint> + <Endpoint name="test-rdms_A9_Infiniband" address="100.74.58.20" protocol="any" isPublic="false" enableDirectServerReturn="false" isDirectAddress="false" disableStealthMode="false"> + <LocalPorts> + <LocalPortSelfManaged /> + </LocalPorts> + </Endpoint> + </InputEndpoints> + </Instance> + </Instances> +</SharedConfig> +""" + +if __name__ == '__main__': + unittest.main() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/tests/upload_status_blob.py new/WALinuxAgent-WALinuxAgent-2.0.12/tests/upload_status_blob.py --- old/WALinuxAgent-WALinuxAgent-2.0.11/tests/upload_status_blob.py 1970-01-01 01:00:00.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/tests/upload_status_blob.py 2015-03-15 15:16:08.000000000 +0100 @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# +# Copyright 2014 Microsoft Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Implements parts of RFC 2131, 1541, 1497 and +# http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx +# http://msdn.microsoft.com/en-us/library/cc227259%28PROT.13%29.aspx +# + +import os +from env import waagent +""" +To run the test, you need to create a file under the same directory called: + status_blob_url.py +and defined the following 2 variables like: + blockBlobUrl="<sas link to a block blob with w/r access>" + pageBlobUrl="<sas link to a page blob with w/r access>" +""" +from status_blob_url import blockBlobUrl, pageBlobUrl + +if __name__ == '__main__': + waagent.LoggerInit('/dev/stdout', '/dev/null', verbose=True) + status = "a" * 512 + waagent.UploadStatusBlob(blockBlobUrl, status.encode("utf-8")) + waagent.UploadStatusBlob(pageBlobUrl, status.encode("utf-8")) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/WALinuxAgent-WALinuxAgent-2.0.11/waagent new/WALinuxAgent-WALinuxAgent-2.0.12/waagent --- old/WALinuxAgent-WALinuxAgent-2.0.11/waagent 2014-12-11 10:04:31.000000000 +0100 +++ new/WALinuxAgent-WALinuxAgent-2.0.12/waagent 2015-03-15 15:16:08.000000000 +0100 @@ -2,7 +2,7 @@ # # Windows Azure Linux Agent # -# Copyright 2014 Microsoft Corporation +# Copyright 2015 Microsoft Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Requires Python 2.4+ and Openssl 1.0+ +# Requires Python 2.6+ and Openssl 1.0+ # # Implements parts of RFC 2131, 1541, 1497 and # http://msdn.microsoft.com/en-us/library/cc227282%28PROT.10%29.aspx @@ -80,7 +80,7 @@ GuestAgentName = "WALinuxAgent" GuestAgentLongName = "Windows Azure Linux Agent" -GuestAgentVersion = "WALinuxAgent-2.0.11" +GuestAgentVersion = "WALinuxAgent-2.0.12" ProtocolVersion = "2012-11-30" #WARNING this value is used to confirm the correct fabric protocol. Config = None @@ -1060,12 +1060,9 @@ def restartSshService(self): """ - Service call to re(start) the SSH service + SSH is socket activated on CoreOS. No need to restart it. """ - retcode = Run("systemctl restart sshd") - if retcode > 0: - Error("Failed to restart SSH service with return code:" + str(retcode)) - return retcode + return 0 def sshDeployPublicKey(self,fprint,path): """ @@ -1374,7 +1371,7 @@ """ Ubuntu specific warning string from Deprovision. """ - print("WARNING! Nameserver configuration in /etc/resolvconf/resolv.conf.d/{tail,originial} will be deleted.") + print("WARNING! Nameserver configuration in /etc/resolvconf/resolv.conf.d/{tail,original} will be deleted.") def deprovisionDeleteFiles(self): """ @@ -1388,7 +1385,7 @@ else: Log("resolvconf is enabled; leaving /etc/resolv.conf intact") resolvConfD = '/etc/resolvconf/resolv.conf.d/' - self.fileBlackList.extend([resolvConfD + 'tail', resolvConfD + 'originial']) + self.fileBlackList.extend([resolvConfD + 'tail', resolvConfD + 'original']) for f in os.listdir(LibDir)+self.fileBlackList: try: os.remove(f) @@ -1788,7 +1785,7 @@ code,output=RunGetOutput("ifconfig",chk_err=False) Log(output) retries=10 - cmd='ifconfig | grep -A1 -B2 ether | grep -B3 inet | grep -A3 UP ' + cmd='ifconfig | grep -A2 -B2 ether | grep -B3 inet | grep -A4 UP ' code=1 while code > 0 : @@ -2593,140 +2590,233 @@ Http communication class. Base of GoalState, and Agent classes. """ - def _HttpGet(self, url, headers): + __RetryWaitingInterval=10 + + def __init__(self): + self.Endpoint = None + + def _ParseUrl(self, url): + secure = False + host = self.Endpoint + action = url + + #Strip "http[s]://hostname/" from url + if url.startswith("http://";): + url = url[7:] + pos = url.index("/") + if pos > 0: + host = url[0: pos] + action = url[pos:] + elif url.startswith("https://";): + secure = True + url = url[8:] + pos = url.index("/") + if pos > 0: + host = url[0:pos] + action = url[pos:] + return host, action, secure + + def _HttpRequest(self, method, host, action, data=None, + secure=False, headers=None): + resp = None; + try: + httpConnection = None + + #If httplib module is not built with ssl support. Failback to http + if secure and hasattr(httplib, "HTTPSConnection"): + httpConnection = httplib.HTTPSConnection(host) + else: + httpConnection = httplib.HTTPConnection(host) + if headers == None: + httpConnection.request(method, action, data) + else: + httpConnection.request(method, action, data, headers) + resp = httpConnection.getresponse() + except httplib.HTTPException, e: + Error('HTTPException {0}, args:{1}'.format(e, repr(e.args))) + except IOError, e: + Error('Socket IOError {0}, args:{1}'.format(e, repr(e.args))) + return resp + + def HttpRequest(self, method, url, data, headers=None, maxRetry=3): """ - Do HTTP get on 'url' with 'headers'. + Sending http request to server On error, sleep 10 and maxRetry times. Return the output buffer or None. """ - LogIfVerbose("HttpGet(" + url + ")") - maxRetry = 2 - if url.startswith("http://";): - url = url[7:] - url = url[url.index("/"):] - for retry in range(0, maxRetry + 1): - strRetry = str(retry) - log = [NoLog, Error][retry > 0] - log("retry HttpGet(" + url + "),retry=" + strRetry) - response = None - strStatus = "None" - try: - httpConnection = httplib.HTTPConnection(self.Endpoint) - if headers == None: - request = httpConnection.request("GET", url) - else: - request = httpConnection.request("GET", url, None, headers) - response = httpConnection.getresponse() - strStatus = str(response.status) - except httplib.HTTPException, e: - Error('HTTPException ' + e.message + ' args: ' + repr(e.args)) - except IOError, e: - Error('socket IOError ' + e.message + ' args: ' + repr(e.args)) - log("response HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if response == None or response.status != httplib.OK: - Error("HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if retry == maxRetry: - Error("return HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return None - else: - Error("sleep 10 seconds HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus) - time.sleep(10) + LogIfVerbose("HTTP Req: {0} {1}".format(method, url)) + LogIfVerbose("HTTP Req: Data={0}".format(data)) + LogIfVerbose("HTTP Req: Header={0}".format(headers)) + host, action, secure = self._ParseUrl(url) + resp = self._HttpRequest(method, host, action, data, secure, headers) + for retry in range(0, maxRetry): + if resp is not None and \ + (resp.status == httplib.OK or \ + resp.status == httplib.CREATED or \ + resp.status == httplib.ACCEPTED): + return resp; + + Error("Retry={0}".format(retry)) + Error("HTTP Req: {0} {1}".format(method, url)) + Error("HTTP Req: Data={0}".format(data)) + Error("HTTP Req: Header={0}".format(headers)) + if resp is None: + Error("HTTP Err: response is empty.".format(retry)) else: - log("return HttpGet(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return response.read() + Error("HTTP Err: Status={0}".format(resp.status)) + Error("HTTP Err: Reason={0}".format(resp.reason)) + Error("HTTP Err: Header={0}".format(resp.getheaders())) + Error("HTTP Err: Body={0}".format(resp.read())) + time.sleep(self.__class__.__RetryWaitingInterval) + resp = self._HttpRequest(method, host, action, data, secure, + headers) + + return None + + def HttpGet(self, url, headers=None, maxRetry=3): + return self.HttpRequest("GET", url, None, headers, maxRetry) + + def HttpHead(self, url, headers=None, maxRetry=3): + return self.HttpRequest("HEAD", url, None, headers, maxRetry) + + def HttpPost(self, url, data, headers=None, maxRetry=3): + return self.HttpRequest("POST", url, data, headers, maxRetry) + + def HttpPut(self, url, data, headers=None, maxRetry=3): + return self.HttpRequest("PUT", url, data, headers, maxRetry) - def HttpGetWithoutHeaders(self, url): + def HttpDelete(url, data, headers=None, maxRetry=3): + return self.HttpRequest("DELETE", url, data, headers, maxRetry) + + def HttpGetWithoutHeaders(self, url, maxRetry=3): """ Return data from an HTTP get on 'url'. """ - return self._HttpGet(url, None) + resp = self.HttpGet(url, None, maxRetry) + return resp.read() if resp is not None else None - def HttpGetWithHeaders(self, url): + def HttpGetWithHeaders(self, url, maxRetry=3): """ Return data from an HTTP get on 'url' with x-ms-agent-name and x-ms-version headers. """ - return self._HttpGet(url, {"x-ms-agent-name": GuestAgentName, "x-ms-version": ProtocolVersion}) + resp = self.HttpGet(url, { + "x-ms-agent-name": GuestAgentName, + "x-ms-version": ProtocolVersion + }, maxRetry) + return resp.read() if resp is not None else None - def HttpSecureGetWithHeaders(self, url, transportCert): + def HttpSecureGetWithHeaders(self, url, transportCert, maxRetry=3): """ Return output of get using ssl cert. """ - return self._HttpGet(url, {"x-ms-agent-name": GuestAgentName, - "x-ms-version": ProtocolVersion, - "x-ms-cipher-name": "DES_EDE3_CBC", - "x-ms-guest-agent-public-x509-cert": transportCert}) - - def HttpPost(self, url, data): - """ - Send http POST to server, sleeping 10 retrying maxRetry times upon error. - """ - LogIfVerbose("HttpPost(" + url + ")") - maxRetry = 2 - for retry in range(0, maxRetry + 1): - strRetry = str(retry) - log = [NoLog, Error][retry > 0] - log("retry HttpPost(" + url + "),retry=" + strRetry) - response = None - strStatus = "None" - try: - httpConnection = httplib.HTTPConnection(self.Endpoint) - request = httpConnection.request("POST", url, data, {"x-ms-agent-name": GuestAgentName, - "Content-Type": "text/xml; charset=utf-8", - "x-ms-version": ProtocolVersion}) - response = httpConnection.getresponse() - strStatus = str(response.status) - except httplib.HTTPException, e: - Error('HTTPException ' + e.message + ' args: ' + repr(e.args)) - except IOError, e: - Error('socket IOError ' + e.message + ' args: ' + repr(e.args)) - log("response HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if response == None or (response.status != httplib.OK and response.status != httplib.ACCEPTED): - Error("HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if retry == maxRetry: - Error("return HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return None - else: - Error("sleep 10 seconds HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus) - time.sleep(10) - else: - log("return HttpPost(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return response + resp = self.HttpGet(url, { + "x-ms-agent-name": GuestAgentName, + "x-ms-version": ProtocolVersion, + "x-ms-cipher-name": "DES_EDE3_CBC", + "x-ms-guest-agent-public-x509-cert": transportCert + }, maxRetry) + return resp.read() if resp is not None else None + + def HttpPostWithHeaders(self, url, data, maxRetry=3): + header = { + "x-ms-agent-name": GuestAgentName, + "Content-Type": "text/xml; charset=utf-8", + "x-ms-version": ProtocolVersion + } + return self.HttpPost(url, data, header, maxRetry) - def HttpPutBlockBlob(self, url, data): - """ - Send http PUT to server, sleeping 10 retrying maxRetry times upon error. - """ - LogIfVerbose("HttpPutBlockBlob(" + url + ")") - maxRetry = 2 - for retry in range(0, maxRetry + 1): - strRetry = str(retry) - log = [NoLog, Error][retry > 0] - log("retry HttpPutBlockBlob(" + url + "),retry=" + strRetry) - response = None - strStatus = "None" - try: - httpConnection = httplib.HTTPConnection(self.Endpoint) - request = httpConnection.request("PUT", url, data, {"x-ms-blob-type" : "BlockBlob", "x-ms-date" : time.strftime("%Y-%M-%dT%H:%M:%SZ", time.gmtime()) ,"Content-Length": str(len(data))} ) - response = httpConnection.getresponse() - strStatus = str(response.status) - except httplib.HTTPException, e: - Error('HTTPException ' + e.message + ' args: ' + repr(e.args)) - except IOError, e: - Error('socket IOError ' + e.message + ' args: ' + repr(e.args)) - log("response HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if response == None or (response.status != httplib.OK and response.status != httplib.CREATED): - Error("HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus) - if retry == maxRetry: - Error("return HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return None - else: - Error("sleep 10 seconds HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus) - time.sleep(10) - else: - log("return HttpPutBlockBlob(" + url + "),retry=" + strRetry + ",status=" + strStatus) - return response +__StorageVersion="2014-02-14" + +def GetBlobType(url): + restutil = Util() + #Check blob type + LogIfVerbose("Check blob type.") + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + blobPropResp = restutil.HttpHead(url, { + "x-ms-date" : timestamp, + 'x-ms-version' : __StorageVersion + }); + blobType = None + if blobPropResp is None: + Error("Can't get status blob type.") + return None + blobType = blobPropResp.getheader("x-ms-blob-type") + LogIfVerbose("Blob type={0}".format(blobType)) + return blobType + +def PutBlockBlob(url, data): + restutil = Util() + LogIfVerbose("Upload block blob") + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + ret = restutil.HttpPut(url, data, { + "x-ms-date" : timestamp, + "x-ms-blob-type" : "BlockBlob", + "Content-Length": str(len(data)), + "x-ms-version" : __StorageVersion + }) + if ret is None: + Error("Failed to upload block blob for status.") + +def PutPageBlob(url, data): + restutil = Util() + LogIfVerbose("Replace old page blob") + timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + #Align to 512 bytes + pageBlobSize = ((len(data) + 511) / 512) * 512 + ret = restutil.HttpPut(url, "", { + "x-ms-date" : timestamp, + "x-ms-blob-type" : "PageBlob", + "Content-Length": "0", + "x-ms-blob-content-length" : str(pageBlobSize), + "x-ms-version" : __StorageVersion + }) + if ret is None: + Error("Failed to clean up page blob for status") + return + + if url.index('?') < 0: + url = "{0}?comp=page".format(url) + else: + url = "{0}&comp=page".format(url) + + LogIfVerbose("Upload page blob") + pageMax = 4 * 1024 * 1024 #Max page size: 4MB + start = 0 + end = 0 + while end < len(data): + end = min(len(data), start + pageMax) + contentSize = end - start + #Align to 512 bytes + pageEnd = ((end + 511) / 512) * 512 + bufSize = pageEnd - start + buf = bytearray(bufSize) + buf[0 : contentSize] = data[start : end] + ret = restutil.HttpPut(url, buf, { + "x-ms-date" : timestamp, + "x-ms-range" : "bytes={0}-{1}".format(start, pageEnd - 1), + "x-ms-page-write" : "update", + "x-ms-version" : __StorageVersion, + "Content-Length": str(pageEnd - start) + }) + if ret is None: + Error("Failed to upload page blob for status") + return + start = end + +def UploadStatusBlob(url, data): + LogIfVerbose("Upload status blob") + LogIfVerbose("Status={0}".format(data)) + blobType = GetBlobType(url) + + if blobType == "BlockBlob": + PutBlockBlob(url, data) + elif blobType == "PageBlob": + PutPageBlob(url, data) + else: + Error("Unknown blob type: {0}".format(blobType)) + return None class TCPHandler(SocketServer.BaseRequestHandler): """ @@ -3032,36 +3122,61 @@ """ Reset members. """ - self.Deployment = None - self.Incarnation = None - self.Role = None - self.LoadBalancerSettings = None - self.OutputEndpoints = None - self.Instances = None + self.RdmaMacAddress = None + self.RdmaIPv4Address = None + self.xmlText = None def Parse(self, xmlText): """ Parse and write configuration to file SharedConfig.xml. """ self.reinitialize() - SetFileContents("SharedConfig.xml", xmlText) + self.xmlText = xmlText dom = xml.dom.minidom.parseString(xmlText) for a in [ "SharedConfig", "Deployment", "Service", "ServiceInstance", "Incarnation", "Role", ]: if not dom.getElementsByTagName(a): Error("SharedConfig.Parse: Missing " + a) - return None + node = dom.childNodes[0] if node.localName != "SharedConfig": Error("SharedConfig.Parse: root not SharedConfig") - return None + + nodes = dom.getElementsByTagName("Instance") + if nodes is not None and len(nodes) != 0: + node = nodes[0] + if node.hasAttribute("rdmaMacAddress"): + self.RdmaMacAddress = node.getAttribute("rdmaMacAddress") + if node.hasAttribute("rdmaIPv4Address"): + self.RdmaIPv4Address = node.getAttribute("rdmaIPv4Address") + return self + + def Save(self): + SetFileContents("SharedConfig.xml", self.xmlText) + + def ConfigRdma(self, dev="/dev/hvnd_rdma"): + if self.RdmaIPv4Address is not None and self.RdmaMacAddress is not None: + if os.path.isfile(dev): + data = ('rdmaMacAddress="{0}" rdmaIPv4Address="{1}"' + '').format(self.RdmaMacAddress, self.RdmaIPv4Address) + Log("Write rdma config to {0}: {1}".format(dev, data)) + try: + with open(dev, "w") as c: + c.write(data) + except IOError, e: + Error("Error writing {0}, {1}".format(dev, e)) + + def InvokeTopologyConsumer(self): program = Config.get("Role.TopologyConsumer") if program != None: try: Children.append(subprocess.Popen([program, LibDir + "/SharedConfig.xml"])) except OSError, e : ErrorWithPrefix('Agent.Run','Exception: '+ str(e) +' occured launching ' + program ) - return self + + def Process(self): + self.ConfigRdma() + self.InvokeTopologyConsumer() class ExtensionsConfig(object): """ @@ -3131,11 +3246,14 @@ try: self.Extensions=dom.getElementsByTagName("Extensions") pg = dom.getElementsByTagName("Plugins") - self.Plugins = pg[0].getElementsByTagName("Plugin") + if len(pg) > 0: + self.Plugins = pg[0].getElementsByTagName("Plugin") + else: + self.Plugins = [] incarnation=self.Extensions[0].getAttribute("goalStateIncarnation") SetFileContents('ExtensionsConfig.'+incarnation+'.xml', xmlText) - except: - LogIfVerbose('ERROR: Error parsing ExtensionsConfig.') + except Exception, e: + Error('ERROR: Error parsing ExtensionsConfig: {0}.'.format(e)) return None for p in self.Plugins: if len(p.getAttribute("location"))<1: # this plugin is inside the PluginSettings @@ -3173,7 +3291,7 @@ Error('Unable to disable '+name) SimpleLog(p.plugin_log,'ERROR: Unable to disable '+name) else : - self.SetHandlerState(handler, 'Installed') + self.SetHandlerState(handler, 'Disabled') Log(name+' is disabled') SimpleLog(p.plugin_log,name+' is disabled') @@ -3608,8 +3726,8 @@ except: Error('Error parsing ExtensionsConfig. Unable to send status reports') return None - self.Util.Endpoint=uri.split('/')[2] - self.Util.HttpPutBlockBlob(uri, status) + + UploadStatusBlob(uri, status.encode("utf-8")) LogIfVerbose('Status report '+status+' sent to ' + uri) return True @@ -3996,6 +4114,8 @@ Calls HostingEnvironmentConfig.Process() """ self.HostingEnvironmentConfig.Process() + self.SharedConfig.Process() + self.SharedConfig.Save() class OvfEnv(object): """ @@ -4056,7 +4176,7 @@ Return self. """ self.reinitialize() - LogIfVerbose(xmlText) + LogIfVerbose(re.sub("<UserPassword>.*?<", "<UserPassword>*<", xmlText)) dom = xml.dom.minidom.parseString(xmlText) if len(dom.getElementsByTagNameNS(self.OvfNs, "Environment")) != 1: Error("Unable to parse OVF XML.") @@ -4411,7 +4531,7 @@ dataFormat = u'<?xml version="1.0"?><TelemetryData version="1.0"><Provider id="{0}">{1}'\ '</Provider></TelemetryData>' data = dataFormat.format(providerid,events) - self.post("/machine/?comp=telemetrydata",data) + self.post("/machine/?comp=telemetrydata", data) def CollectAndSendWALAEvents(self): if not os.path.exists(self.eventdir): @@ -4893,7 +5013,7 @@ + "</ContainerId><RoleInstanceList><Role><InstanceId>" + self.GoalState.RoleInstanceId + "</InstanceId><Health><State>Ready</State></Health></Role></RoleInstanceList></Container></Health>") - a = self.HttpPost("/machine?comp=health", healthReport) + a = self.HttpPostWithHeaders("/machine?comp=health", healthReport) if a != None: return a.getheader("x-ms-latest-goal-state-incarnation-number") return None @@ -4912,7 +5032,7 @@ + "</InstanceId><Health><State>NotReady</State>" + "<Details><SubStatus>" + status + "</SubStatus><Description>" + desc + "</Description></Details>" + "</Health></Role></RoleInstanceList></Container></Health>") - a = self.HttpPost("/machine?comp=health", healthReport) + a = self.HttpPostWithHeaders("/machine?comp=health", healthReport) if a != None: return a.getheader("x-ms-latest-goal-state-incarnation-number") return None @@ -4927,7 +5047,8 @@ + "<Id>" + self.GoalState.RoleInstanceId + "</Id>" + "<Properties><Property name=\"CertificateThumbprint\" value=\"" + thumbprint + "\" /></Properties>" + "</RoleInstance></RoleInstances></Container></RoleProperties>") - a = self.HttpPost("/machine?comp=roleProperties", roleProperties) + a = self.HttpPostWithHeaders("/machine?comp=roleProperties", + roleProperties) Log("Posted Role Properties. CertificateThumbprint=" + thumbprint) return a @@ -5295,7 +5416,7 @@ goalState.ExtensionsConfig.ReportHandlerStatus() if not eventMonitor: - eventMonitor = WALAEventMonitor(self.HttpPost) + eventMonitor = WALAEventMonitor(self.HttpPostWithHeaders) eventMonitor.StartEventsLoop() time.sleep(25 - sleepToReduceAccessDenied)
