On x86_64 systems, this test inmate measures the time that is required to read a value from main memory. Via rdtsc, it measures the CPU cycles that are required for the access. Access can either happen cached, or uncached. In case of uncached access, the cache line will be flushed before access.
This tool repeats the measurement for 10e6 times, and outputs the average cycles that were required for the access. Before accessing the actual measurement, a dummy test is used to determine the average overhead of one single measurement. And that's pretty useful, because this tool gives a lot of insights of differences between the root and the non-root cell: With tiny effort, we can also run it on Linux. If the 'overhead' time differs between root and non-root cell, this can be an indicator that there might be some timing or speed differences between the root and non-root cell. If the 'uncached' or 'cached' average time differs between the non-root and root cell, it's an indicator that both might have different hardware configurations / setups. Signed-off-by: Ralf Ramsauer <[email protected]> --- since v1: - Move host code to tools/ since RFC: - move the inmate to demos instead of tests .gitignore | 1 + inmates/demos/x86/Makefile | 4 +- inmates/demos/x86/cache-timings-common.c | 95 ++++++++++++++++++++++++ inmates/demos/x86/cache-timings.c | 15 ++++ tools/Makefile | 9 ++- tools/cache-timings.c | 29 ++++++++ 6 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 inmates/demos/x86/cache-timings-common.c create mode 100644 inmates/demos/x86/cache-timings.c create mode 100644 tools/cache-timings.c diff --git a/.gitignore b/.gitignore index 4691ff79..89248c17 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ hypervisor/hypervisor.lds inmates/lib/arm/inmate.lds inmates/lib/arm64/inmate.lds pyjailhouse/pci_defs.py +tools/cache-timings tools/ivshmem-demo tools/jailhouse tools/jailhouse-gcov-extract diff --git a/inmates/demos/x86/Makefile b/inmates/demos/x86/Makefile index f53b739e..47b79869 100644 --- a/inmates/demos/x86/Makefile +++ b/inmates/demos/x86/Makefile @@ -13,7 +13,8 @@ include $(INMATES_LIB)/Makefile.lib INMATES := tiny-demo.bin apic-demo.bin ioapic-demo.bin 32-bit-demo.bin \ - pci-demo.bin e1000-demo.bin ivshmem-demo.bin smp-demo.bin + pci-demo.bin e1000-demo.bin ivshmem-demo.bin smp-demo.bin \ + cache-timings.bin tiny-demo-y := tiny-demo.o apic-demo-y := apic-demo.o @@ -22,6 +23,7 @@ pci-demo-y := pci-demo.o e1000-demo-y := e1000-demo.o ivshmem-demo-y := ../ivshmem-demo.o smp-demo-y := smp-demo.o +cache-timings-y := cache-timings.o $(eval $(call DECLARE_32_BIT,32-bit-demo)) 32-bit-demo-y := 32-bit-demo.o diff --git a/inmates/demos/x86/cache-timings-common.c b/inmates/demos/x86/cache-timings-common.c new file mode 100644 index 00000000..0edf65e6 --- /dev/null +++ b/inmates/demos/x86/cache-timings-common.c @@ -0,0 +1,95 @@ +/* + * Jailhouse, a Linux-based partitioning hypervisor + * + * Copyright (c) OTH Regensburg, 2020 + * + * Authors: + * Ralf Ramsauer <[email protected]> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#define ROUNDS (10 * 1000 * 1000) + +union tscval { + struct { + u32 lo; + u32 hi; + } __attribute__((packed)); + u64 val; +} __attribute__((packed)); + +static u32 victim; + +static inline void clflush(void *addr) +{ + asm volatile("clflush %0\t\n" + "mfence\t\n" + "lfence\t\n" : "+m" (*(volatile char *)addr)); +} + +#define MEASUREMENT_OVERHEAD "nop\t\n" +#define MEASUREMENT_COMMAND "mov (%%rbx), %%ebx\t\n" +#define DECLARE_MEASUREMENT(name, flush, meas) \ + static inline u64 measure_##name(u32 *victim) \ + { \ + union tscval before, after; \ + \ + if (flush) \ + clflush(victim); \ + asm volatile("mov %4, %%rbx\t\n" \ + "lfence\t\n" \ + "rdtsc\t\n" \ + "lfence\t\n" \ + \ + meas \ + \ + "mov %%eax, %%ebx\t\n" \ + "mov %%edx, %%ecx\t\n" \ + "lfence\t\n" \ + "rdtsc\t\n" \ + "lfence\t\n" \ + "mov %%ebx, %0\t\n" \ + "mov %%ecx, %1\t\n" \ + "mov %%eax, %2\t\n" \ + "mov %%edx, %3\t\n" \ + : "=m"(before.lo), "=m" (before.hi), \ + "=m" (after.lo), "=m" (after.hi) \ + : "m" (victim) \ + : "eax", "rbx", "ecx", "edx"); \ + return after.val - before.val; \ + } + +DECLARE_MEASUREMENT(overhead, false, MEASUREMENT_OVERHEAD) +DECLARE_MEASUREMENT(cached, false, MEASUREMENT_COMMAND) +DECLARE_MEASUREMENT(uncached, true, MEASUREMENT_COMMAND) + +static inline u64 avg_measurement(u64 (*meas)(u32*), u32 *victim, + unsigned int rounds, u64 overhead) +{ + u64 cycles = 0; + unsigned int i; + + for (i = 0; i < rounds; i++) + cycles += meas(victim) - overhead; + return cycles / rounds; +} + +void inmate_main(void) +{ + u64 cycles, overhead; + + printk("Measurement rounds: %u\n", ROUNDS); + printk("Determining measurement overhead...\n"); + overhead = avg_measurement(measure_overhead, &victim, ROUNDS, 0); + printk(" -> Average measurement overhead: %llu cycles\n", overhead); + + printk("Measuring uncached memory access...\n"); + cycles = avg_measurement(measure_uncached, &victim, ROUNDS, overhead); + printk(" -> Average uncached memory access: %llu cycles\n", cycles); + + printk("Measuring cached memory access...\n"); + cycles = avg_measurement(measure_cached, &victim, ROUNDS, overhead); + printk(" -> Average cached memory access: %llu cycles\n", cycles); +} diff --git a/inmates/demos/x86/cache-timings.c b/inmates/demos/x86/cache-timings.c new file mode 100644 index 00000000..1acc3ee9 --- /dev/null +++ b/inmates/demos/x86/cache-timings.c @@ -0,0 +1,15 @@ +/* + * Jailhouse, a Linux-based partitioning hypervisor + * + * Copyright (c) OTH Regensburg, 2020 + * + * Authors: + * Ralf Ramsauer <[email protected]> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include <inmate.h> + +#include "cache-timings-common.c" diff --git a/tools/Makefile b/tools/Makefile index e6945cd6..33a39d26 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -35,6 +35,13 @@ KBUILD_CFLAGS += $(call cc-option, -fno-pie) KBUILD_CFLAGS += $(call cc-option, -no-pie) BINARIES := jailhouse ivshmem-demo +targets += jailhouse.o ivshmem-demo.o + +ifeq ($(ARCH),x86) +BINARIES += cache-timings +targets += cache-timings.o +endif # $(ARCH),x86 + always-y := $(BINARIES) HAS_PYTHON_MAKO := \ @@ -104,8 +111,6 @@ define cmd_gen_man sed 's/$${VERSION}/$(shell cat $(src)/../VERSION)/g' $< > $@ endef -targets += jailhouse.o ivshmem-demo.o - $(obj)/%: $(obj)/%.o $(call if_changed,ld) diff --git a/tools/cache-timings.c b/tools/cache-timings.c new file mode 100644 index 00000000..2c591dab --- /dev/null +++ b/tools/cache-timings.c @@ -0,0 +1,29 @@ +/* + * Jailhouse, a Linux-based partitioning hypervisor + * + * Copyright (c) OTH Regensburg, 2020 + * + * Authors: + * Ralf Ramsauer <[email protected]> + * + * This work is licensed under the terms of the GNU GPL, version 2. See + * the COPYING file in the top-level directory. + */ + +#include <stdbool.h> +#include <stdio.h> + +#define printk printf + +typedef unsigned int u32; +typedef unsigned long long u64; + +void inmate_main(void); + +#include "../inmates/demos/x86/cache-timings-common.c" + +int main(void) +{ + inmate_main(); + return 0; +} -- 2.29.1 -- You received this message because you are subscribed to the Google Groups "Jailhouse" group. To unsubscribe from this group and stop receiving emails from it, send an email to [email protected]. To view this discussion on the web visit https://groups.google.com/d/msgid/jailhouse-dev/20201027214021.407707-2-ralf.ramsauer%40oth-regensburg.de.
