Francesco Romani has uploaded a new change for review. Change subject: sampling: move translation code into hoststats.py ......................................................................
sampling: move translation code into hoststats.py Move all the functions that produces host stats from HostSamples into hoststats.py, like we did for vmstats.py. Note: getBootTime() and get_boot_time() were moved as well, even though they aren't translation functions. Should they stay into sampling.py? Change-Id: Icce6acbd596d087491678d039282d5d37d761904 Signed-off-by: Francesco Romani <[email protected]> --- M debian/vdsm.install M tests/samplingTests.py M vdsm.spec.in M vdsm/virt/Makefile.am A vdsm/virt/hoststats.py M vdsm/virt/sampling.py 6 files changed, 222 insertions(+), 185 deletions(-) git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/34/42034/1 diff --git a/debian/vdsm.install b/debian/vdsm.install index ae35369..5d01712 100644 --- a/debian/vdsm.install +++ b/debian/vdsm.install @@ -148,6 +148,7 @@ ./usr/share/vdsm/virt/__init__.py ./usr/share/vdsm/virt/domain_descriptor.py ./usr/share/vdsm/virt/guestagent.py +./usr/share/vdsm/virt/hoststats.py ./usr/share/vdsm/virt/migration.py ./usr/share/vdsm/virt/periodic.py ./usr/share/vdsm/virt/sampling.py diff --git a/tests/samplingTests.py b/tests/samplingTests.py index 229d3d4..ab85834 100644 --- a/tests/samplingTests.py +++ b/tests/samplingTests.py @@ -28,7 +28,9 @@ from vdsm import ipwrapper from vdsm.password import ProtectedPassword +from virt import hoststats import virt.sampling as sampling + import caps @@ -80,39 +82,39 @@ shutil.rmtree(self._tmpDir) def testBootTimeOk(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', self._good_path)]): - self.assertEquals(sampling.getBootTime(), + self.assertEquals(hoststats.getBootTime(), 1395249141) def testBootTimeEmpty(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', '/dev/null')]): with self.assertRaises(ValueError): - sampling.getBootTime() + hoststats.getBootTime() def testBootTimeMissing(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', self._missing_path)]): with self.assertRaises(ValueError): - sampling.getBootTime() + hoststats.getBootTime() def testBootTimeMalformed(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', self._malformed_path)]): with self.assertRaises(ValueError): - sampling.getBootTime() + hoststats.getBootTime() def testBootTimeNonExistantFile(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', '/i/do/not/exist/1234567890')]): with self.assertRaises(IOError): - sampling.getBootTime() + hoststats.getBootTime() def testBootTimeExtra(self): - with MonkeyPatchScope([(sampling, '_PROC_STAT_PATH', + with MonkeyPatchScope([(hoststats, '_PROC_STAT_PATH', self._extra_path)]): - self.assertEquals(sampling.getBootTime(), 1395249141) + self.assertEquals(hoststats.getBootTime(), 1395249141) @contextmanager @@ -336,7 +338,7 @@ with MonkeyPatchScope([(caps, 'getNumaTopology', fakeNumaTopology)]): self.assertEqual( - sampling._get_cpu_core_stats(first_sample, last_sample), + hoststats._get_cpu_core_stats(first_sample, last_sample), expected) diff --git a/vdsm.spec.in b/vdsm.spec.in index 5c9061d..7d7abd2 100644 --- a/vdsm.spec.in +++ b/vdsm.spec.in @@ -1000,6 +1000,7 @@ %{_datadir}/%{vdsm_name}/virt/__init__.py* %{_datadir}/%{vdsm_name}/virt/domain_descriptor.py* %{_datadir}/%{vdsm_name}/virt/guestagent.py* +%{_datadir}/%{vdsm_name}/virt/hoststats.py* %{_datadir}/%{vdsm_name}/virt/migration.py* %{_datadir}/%{vdsm_name}/virt/periodic.py* %{_datadir}/%{vdsm_name}/virt/sampling.py* diff --git a/vdsm/virt/Makefile.am b/vdsm/virt/Makefile.am index 376f973..88025be 100644 --- a/vdsm/virt/Makefile.am +++ b/vdsm/virt/Makefile.am @@ -27,6 +27,7 @@ __init__.py \ domain_descriptor.py \ guestagent.py \ + hoststats.py \ migration.py \ periodic.py \ sampling.py \ diff --git a/vdsm/virt/hoststats.py b/vdsm/virt/hoststats.py new file mode 100644 index 0000000..57c6954 --- /dev/null +++ b/vdsm/virt/hoststats.py @@ -0,0 +1,202 @@ +# +# Copyright 2008-2015 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +# +# Refer to the README and COPYING files for full details of the license +# + +import logging + +from vdsm import utils + +import caps +import v2v + + +def produce(ncpus, first_sample, last_sample): + stats = _empty_stats() + + if first_sample is None: + return stats + + stats.update(_get_interfaces_stats(first_sample, last_sample)) + + interval = last_sample.timestamp - first_sample.timestamp + + jiffies = ( + last_sample.pidcpu.user - first_sample.pidcpu.user) % (2 ** 32) + stats['cpuUserVdsmd'] = jiffies / interval + jiffies = ( + last_sample.pidcpu.sys - first_sample.pidcpu.sys) % (2 ** 32) + stats['cpuSysVdsmd'] = jiffies / interval + + jiffies = ( + last_sample.totcpu.user - first_sample.totcpu.user) % (2 ** 32) + stats['cpuUser'] = jiffies / interval / ncpus + jiffies = ( + last_sample.totcpu.sys - first_sample.totcpu.sys) % (2 ** 32) + stats['cpuSys'] = jiffies / interval / ncpus + stats['cpuIdle'] = max(0.0, + 100.0 - stats['cpuUser'] - stats['cpuSys']) + stats['memUsed'] = last_sample.memUsed + stats['anonHugePages'] = last_sample.anonHugePages + stats['cpuLoad'] = last_sample.cpuLoad + + stats['diskStats'] = last_sample.diskStats + stats['thpState'] = last_sample.thpState + + if _boot_time(): + stats['bootTime'] = _boot_time() + + stats['numaNodeMemFree'] = last_sample.numaNodeMem.nodesMemSample + stats['cpuStatistics'] = _get_cpu_core_stats( + first_sample, last_sample) + + stats['v2vJobs'] = v2v.get_jobs_status() + return stats + + +_PROC_STAT_PATH = '/proc/stat' + + +def getBootTime(): + """ + Returns the boot time of the machine in seconds since epoch. + + Raises IOError if file access fails, or ValueError if boot time not + present in file. + """ + with open(_PROC_STAT_PATH) as proc_stat: + for line in proc_stat: + if line.startswith('btime'): + parts = line.split() + if len(parts) > 1: + return int(parts[1]) + else: + break + raise ValueError('Boot time not present') + + [email protected] +def _boot_time(): + # Try to get boot time only once, if N/A just log the error and never + # include it in the response. + try: + return getBootTime() + except (IOError, ValueError): + logging.exception('Failed to get boot time') + return None + + +def _empty_stats(): + return { + 'cpuUser': 0.0, + 'cpuSys': 0.0, + 'cpuIdle': 100.0, + 'rxRate': 0.0, # REQUIRED_FOR: engine < 3.6 + 'txRate': 0.0, # REQUIRED_FOR: engine < 3.6 + 'cpuSysVdsmd': 0.0, + 'cpuUserVdsmd': 0.0, + } + + +def _get_cpu_core_stats(first_sample, last_sample): + interval = last_sample.timestamp - first_sample.timestamp + + def compute_cpu_usage(cpu_core, mode): + jiffies = ( + last_sample.cpuCores.getCoreSample(cpu_core)[mode] - + first_sample.cpuCores.getCoreSample(cpu_core)[mode] + ) % (2 ** 32) + return ("%.2f" % (jiffies / interval)) + + cpu_core_stats = {} + for node_index, numa_node in caps.getNumaTopology().iteritems(): + cpu_cores = numa_node['cpus'] + for cpu_core in cpu_cores: + core_stat = { + 'nodeIndex': int(node_index), + 'cpuUser': compute_cpu_usage(cpu_core, 'user'), + 'cpuSys': compute_cpu_usage(cpu_core, 'sys'), + } + core_stat['cpuIdle'] = ( + "%.2f" % max(0.0, + 100.0 - + float(core_stat['cpuUser']) - + float(core_stat['cpuSys']))) + cpu_core_stats[str(cpu_core)] = core_stat + return cpu_core_stats + + +def _get_interfaces_stats(first_sample, last_sample): + interval = last_sample.timestamp - first_sample.timestamp + + rx = tx = rxDropped = txDropped = 0 + stats = {'network': {}} + total_rate = 0 + for ifid in last_sample.interfaces: + # it skips hot-plugged devices if we haven't enough information + # to count stats from it + if ifid not in first_sample.interfaces: + continue + + ifrate = last_sample.interfaces[ifid].speed or 1000 + Mbps2Bps = (10 ** 6) / 8 + thisRx = ( + last_sample.interfaces[ifid].rx - + first_sample.interfaces[ifid].rx + ) % (2 ** 32) + thisTx = ( + last_sample.interfaces[ifid].tx - + first_sample.interfaces[ifid].tx + ) % (2 ** 32) + rxRate = 100.0 * thisRx / interval / ifrate / Mbps2Bps + txRate = 100.0 * thisTx / interval / ifrate / Mbps2Bps + if txRate > 100 or rxRate > 100: + txRate = min(txRate, 100.0) + rxRate = min(rxRate, 100.0) + logging.debug('Rate above 100%%.') + iface = last_sample.interfaces[ifid] + stats['network'][ifid] = { + 'name': ifid, 'speed': str(ifrate), + 'rxDropped': str(iface.rxDropped), + 'txDropped': str(iface.txDropped), + 'rxErrors': str(iface.rxErrors), + 'txErrors': str(iface.txErrors), + 'state': iface.operstate, + 'rxRate': '%.1f' % rxRate, + 'txRate': '%.1f' % txRate, + 'rx': str(iface.rx), + 'tx': str(iface.tx), + 'sampleTime': last_sample.timestamp, + } + rx += thisRx + tx += thisTx + rxDropped += last_sample.interfaces[ifid].rxDropped + txDropped += last_sample.interfaces[ifid].txDropped + total_rate += ifrate + + total_bytes_per_sec = (total_rate or 1000) * (10 ** 6) / 8 + stats['rxRate'] = 100.0 * rx / interval / total_bytes_per_sec + stats['txRate'] = 100.0 * tx / interval / total_bytes_per_sec + if stats['txRate'] > 100 or stats['rxRate'] > 100: + stats['txRate'] = min(stats['txRate'], 100.0) + stats['rxRate'] = min(stats['rxRate'], 100.0) + logging.debug(stats) + stats['rxDropped'] = rxDropped + stats['txDropped'] = txDropped + + return stats diff --git a/vdsm/virt/sampling.py b/vdsm/virt/sampling.py index 38c5892..5729a54 100644 --- a/vdsm/virt/sampling.py +++ b/vdsm/virt/sampling.py @@ -35,9 +35,9 @@ from vdsm import netinfo from vdsm import utils from vdsm.config import config -import v2v from .utils import ExpiringCache +from . import hoststats import caps @@ -196,27 +196,6 @@ class TimedSample(object): def __init__(self): self.timestamp = time.time() - - -_PROC_STAT_PATH = '/proc/stat' - - -def getBootTime(): - """ - Returns the boot time of the machine in seconds since epoch. - - Raises IOError if file access fails, or ValueError if boot time not - present in file. - """ - with open(_PROC_STAT_PATH) as proc_stat: - for line in proc_stat: - if line.startswith('btime'): - parts = line.split() - if len(parts) > 1: - return int(parts[1]) - else: - break - raise ValueError('Boot time not present') def _get_interfaces_and_samples(): @@ -590,158 +569,9 @@ if not self._stopEvent.isSet(): self._log.exception("Error while sampling stats") - @utils.memoized - def _boot_time(self): - # Try to get boot time only once, if N/A just log the error and never - # include it in the response. - try: - return getBootTime() - except (IOError, ValueError): - self._log.exception('Failed to get boot time') - return None - def get(self): - stats = self._empty_stats() - first_sample, last_sample, _ = self._samples.stats() - if first_sample is None: - return stats - - stats.update(_get_interfaces_stats(first_sample, last_sample)) - - interval = last_sample.timestamp - first_sample.timestamp - - jiffies = ( - last_sample.pidcpu.user - first_sample.pidcpu.user) % (2 ** 32) - stats['cpuUserVdsmd'] = jiffies / interval - jiffies = ( - last_sample.pidcpu.sys - first_sample.pidcpu.sys) % (2 ** 32) - stats['cpuSysVdsmd'] = jiffies / interval - - jiffies = ( - last_sample.totcpu.user - first_sample.totcpu.user) % (2 ** 32) - stats['cpuUser'] = jiffies / interval / self._ncpus - jiffies = ( - last_sample.totcpu.sys - first_sample.totcpu.sys) % (2 ** 32) - stats['cpuSys'] = jiffies / interval / self._ncpus - stats['cpuIdle'] = max(0.0, - 100.0 - stats['cpuUser'] - stats['cpuSys']) - stats['memUsed'] = last_sample.memUsed - stats['anonHugePages'] = last_sample.anonHugePages - stats['cpuLoad'] = last_sample.cpuLoad - - stats['diskStats'] = last_sample.diskStats - stats['thpState'] = last_sample.thpState - - if self._boot_time(): - stats['bootTime'] = self._boot_time() - - stats['numaNodeMemFree'] = last_sample.numaNodeMem.nodesMemSample - stats['cpuStatistics'] = _get_cpu_core_stats( - first_sample, last_sample) - - stats['v2vJobs'] = v2v.get_jobs_status() - return stats - - def _empty_stats(self): - return { - 'cpuUser': 0.0, - 'cpuSys': 0.0, - 'cpuIdle': 100.0, - 'rxRate': 0.0, # REQUIRED_FOR: engine < 3.6 - 'txRate': 0.0, # REQUIRED_FOR: engine < 3.6 - 'cpuSysVdsmd': 0.0, - 'cpuUserVdsmd': 0.0, - } - - -def _get_cpu_core_stats(first_sample, last_sample): - interval = last_sample.timestamp - first_sample.timestamp - - def compute_cpu_usage(cpu_core, mode): - jiffies = ( - last_sample.cpuCores.getCoreSample(cpu_core)[mode] - - first_sample.cpuCores.getCoreSample(cpu_core)[mode] - ) % (2 ** 32) - return ("%.2f" % (jiffies / interval)) - - cpu_core_stats = {} - for node_index, numa_node in caps.getNumaTopology().iteritems(): - cpu_cores = numa_node['cpus'] - for cpu_core in cpu_cores: - core_stat = { - 'nodeIndex': int(node_index), - 'cpuUser': compute_cpu_usage(cpu_core, 'user'), - 'cpuSys': compute_cpu_usage(cpu_core, 'sys'), - } - core_stat['cpuIdle'] = ( - "%.2f" % max(0.0, - 100.0 - - float(core_stat['cpuUser']) - - float(core_stat['cpuSys']))) - cpu_core_stats[str(cpu_core)] = core_stat - return cpu_core_stats - - -def _get_interfaces_stats(first_sample, last_sample): - interval = last_sample.timestamp - first_sample.timestamp - - rx = tx = rxDropped = txDropped = 0 - stats = {'network': {}} - total_rate = 0 - for ifid in last_sample.interfaces: - # it skips hot-plugged devices if we haven't enough information - # to count stats from it - if ifid not in first_sample.interfaces: - continue - - ifrate = last_sample.interfaces[ifid].speed or 1000 - Mbps2Bps = (10 ** 6) / 8 - thisRx = ( - last_sample.interfaces[ifid].rx - - first_sample.interfaces[ifid].rx - ) % (2 ** 32) - thisTx = ( - last_sample.interfaces[ifid].tx - - first_sample.interfaces[ifid].tx - ) % (2 ** 32) - rxRate = 100.0 * thisRx / interval / ifrate / Mbps2Bps - txRate = 100.0 * thisTx / interval / ifrate / Mbps2Bps - if txRate > 100 or rxRate > 100: - txRate = min(txRate, 100.0) - rxRate = min(rxRate, 100.0) - logging.debug('Rate above 100%%.') - iface = last_sample.interfaces[ifid] - stats['network'][ifid] = { - 'name': ifid, 'speed': str(ifrate), - 'rxDropped': str(iface.rxDropped), - 'txDropped': str(iface.txDropped), - 'rxErrors': str(iface.rxErrors), - 'txErrors': str(iface.txErrors), - 'state': iface.operstate, - 'rxRate': '%.1f' % rxRate, - 'txRate': '%.1f' % txRate, - 'rx': str(iface.rx), - 'tx': str(iface.tx), - 'sampleTime': last_sample.timestamp, - } - rx += thisRx - tx += thisTx - rxDropped += last_sample.interfaces[ifid].rxDropped - txDropped += last_sample.interfaces[ifid].txDropped - total_rate += ifrate - - total_bytes_per_sec = (total_rate or 1000) * (10 ** 6) / 8 - stats['rxRate'] = 100.0 * rx / interval / total_bytes_per_sec - stats['txRate'] = 100.0 * tx / interval / total_bytes_per_sec - if stats['txRate'] > 100 or stats['rxRate'] > 100: - stats['txRate'] = min(stats['txRate'], 100.0) - stats['rxRate'] = min(stats['rxRate'], 100.0) - logging.debug(stats) - stats['rxDropped'] = rxDropped - stats['txDropped'] = txDropped - - return stats + return hoststats.produce(self._ncpus, first_sample, last_sample) def _getLinkSpeed(dev): -- To view, visit https://gerrit.ovirt.org/42034 To unsubscribe, visit https://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Icce6acbd596d087491678d039282d5d37d761904 Gerrit-PatchSet: 1 Gerrit-Project: vdsm Gerrit-Branch: master Gerrit-Owner: Francesco Romani <[email protected]> _______________________________________________ vdsm-patches mailing list [email protected] https://lists.fedorahosted.org/mailman/listinfo/vdsm-patches
