[RFC PATCH 3/3] restartable sequences: basic user-space self-tests
Implements basic tests of RSEQ functionality.
"basic_percpu_ops_test" implements a few simple per-cpu operations and
testing their correctness.
---
tools/testing/selftests/rseq/Makefile | 14 +
.../testing/selftests/rseq/basic_percpu_ops_test.c | 331 +
tools/testing/selftests/rseq/rseq.c| 48 +++
tools/testing/selftests/rseq/rseq.h| 17 ++
4 files changed, 410 insertions(+)
create mode 100644 tools/testing/selftests/rseq/Makefile
create mode 100644 tools/testing/selftests/rseq/basic_percpu_ops_test.c
create mode 100644 tools/testing/selftests/rseq/rseq.c
create mode 100644 tools/testing/selftests/rseq/rseq.h
diff --git a/tools/testing/selftests/rseq/Makefile
b/tools/testing/selftests/rseq/Makefile
new file mode 100644
index 000..3a9cb5c
--- /dev/null
+++ b/tools/testing/selftests/rseq/Makefile
@@ -0,0 +1,14 @@
+CFLAGS += -Wall
+LDFLAGS += -lpthread
+
+TESTS = basic_test basic_percpu_ops_test
+
+basic_percpu_ops_test: basic_percpu_ops_test.c
+
+
+all: $(TESTS)
+%: %.c
+ $(CC) $(CFLAGS) -o $@ $^ rseq.c $(LDFLAGS)
+
+clean:
+ $(RM) $(TESTS)
diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c
b/tools/testing/selftests/rseq/basic_percpu_ops_test.c
new file mode 100644
index 000..63a668d
--- /dev/null
+++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c
@@ -0,0 +1,331 @@
+#define _GNU_SOURCE
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "rseq.h"
+
+#if defined(__x86_64__)
+
+#define barrier() {__asm__ __volatile__("" : : : "memory"); }
+
+struct rseq_section {
+ void *begin;
+ void *end;
+ void *restart;
+};
+
+extern struct rseq_section const __start___rseq_sections[]
+__attribute((weak));
+extern struct rseq_section const __stop___rseq_sections[]
+__attribute((weak));
+
+/* Implemented by percpu_ops.S */
+struct percpu_lock {
+ int word[CPU_SETSIZE][16]; /* cache aligned; lock-word is [cpu][0] */
+};
+
+/* A simple percpu spinlock. Returns the cpu lock was acquired on. */
+int rseq_percpu_lock(struct percpu_lock *lock)
+{
+ int out = -1;
+
+ asm volatile (
+ "1:\n\t"
+ "movl %1, %0\n\t"
+ "leaq (,%0,8), %%r10\n\t"
+ "leaq (%2, %%r10, 8), %%r10\n\t"
+ "2:\n\t"
+ "cmpl $0, (%%r10)\n\t"
+ "jne 2b\n\t"
+ "movl $1, (%%r10)\n\t"
+ "3:\n\t"
+ ".pushsection __rseq_sections, \"a\"\n\t"
+ ".quad 1b, 3b, 1b\n\t"
+ ".popsection\n\t"
+ : "+r" (out)
+ : "m" (__rseq_current_cpu), "r" ((unsigned long)lock)
+ : "memory", "r10");
+ return out;
+}
+
+/*
+ * cmpxchg [with an additional check value].
+ *
+ * Returns:
+ * -1 if *p != old or cpu != current cpu [ || check_ptr != check_val, ]
+ * otherwise 0.
+ *
+ * Note: When specified, check_ptr is dereferenced iff *p == old
+ */
+int rseq_percpu_cmpxchg(int cpu, intptr_t *p, intptr_t old, intptr_t new)
+{
+ asm volatile goto (
+ "1:\n\t"
+ "cmpl %1, %0\n\t"
+ "jne %l[fail]\n\t"
+ "cmpq %2, %3\n\t"
+ "jne %l[fail]\n\t"
+ "movq %4, %3\n\t"
+ "2:\n\t"
+ ".pushsection __rseq_sections, \"a\"\n\t"
+ ".quad 1b, 2b, 1b\n\t"
+ ".popsection\n\t"
+ :
+ : "r" (cpu), "m" (__rseq_current_cpu),
+ "r" (old), "m" (*p), "r" (new)
+ : "memory"
+ : fail);
+ return 0;
+fail:
+ return -1;
+}
+int rseq_percpu_cmpxchgcheck(int cpu, intptr_t *p, intptr_t old, intptr_t new,
+ intptr_t *check_ptr, intptr_t check_val)
+{
+ asm volatile goto (
+ "1:\n\t"
+ "cmpl %1, %0\n\t"
+ "jne %l[fail]\n\t"
+ "cmpq %2, %3\n\t"
+ "jne %l[fail]\n\t"
+ "cmpq %5, %6\n\t"
+ "jne %l[fail]\n\t"
+ "movq %4, %3\n\t"
+ "2:\n\t"
+ ".pushsection __rseq_sections, \"a\"\n\t"
+ ".quad 1b, 2b, 1b\n\t"
+ ".popsection\n\t"
+ :
+ : "r" (cpu), "m" (__rseq_current_cpu),
+ "r" (old), "m" (*p), "r" (new),
+ "r" (check_val), "m" (*check_ptr)
+ : "memory"
+ : fail);
+ return 0;
+fail:
+ return -1;
+}
+
+
+void rseq_percpu_unlock(struct percpu_lock *lock, int cpu)
+{
+ barrier(); /* need a release-store here, this suffices on x86. */
+ assert(lock->word[cpu][0] == 1);
+ lock->word[cpu][0] = 0;
+}
+
+void rseq_unknown_restart_addr(void *addr)
+{
+ fprintf(stderr, "rseq: unrecognized restart address %p\n", addr);
+ exit(1);
+}
+
+struct spinlock_test_data {
+ struct percpu_lock
[RFC PATCH 3/3] restartable sequences: basic user-space self-tests
Implements two basic tests of RSEQ functionality. The first, "basic_test" only asserts that RSEQ works moderately correctly. E.g. that: - The CPUID pointer works - Code infinitely looping within a critical section will eventually be interrupted. "basic_percpu_ops_test" is a slightly more "realistic" variant, implementing a few simple per-cpu operations and testing their correctness. It also includes a trivial example of user-space may multiplexing the critical section via the restart handler. Signed-off-by: Paul Turner --- tools/testing/selftests/rseq/Makefile | 15 + .../testing/selftests/rseq/basic_percpu_ops_test.S | 131 ++ .../testing/selftests/rseq/basic_percpu_ops_test.c | 250 tools/testing/selftests/rseq/basic_test.c | 76 ++ tools/testing/selftests/rseq/rseq.c| 48 tools/testing/selftests/rseq/rseq.h| 28 ++ 6 files changed, 548 insertions(+) create mode 100644 tools/testing/selftests/rseq/Makefile create mode 100644 tools/testing/selftests/rseq/basic_percpu_ops_test.S create mode 100644 tools/testing/selftests/rseq/basic_percpu_ops_test.c create mode 100644 tools/testing/selftests/rseq/basic_test.c create mode 100644 tools/testing/selftests/rseq/rseq.c create mode 100644 tools/testing/selftests/rseq/rseq.h diff --git a/tools/testing/selftests/rseq/Makefile b/tools/testing/selftests/rseq/Makefile new file mode 100644 index 000..c5a2b47 --- /dev/null +++ b/tools/testing/selftests/rseq/Makefile @@ -0,0 +1,15 @@ +CFLAGS += -Wall +LDFLAGS += -lpthread + +TESTS = basic_test basic_percpu_ops_test + +basic_percpu_ops_test: basic_percpu_ops_test.c basic_percpu_ops_test.S + +all: $(TESTS) +%: %.c + $(CC) $(CFLAGS) -o $@ $^ rseq.c $(LDFLAGS) + +include ../lib.mk + +clean: + $(RM) $(TESTS) diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.S b/tools/testing/selftests/rseq/basic_percpu_ops_test.S new file mode 100644 index 000..7da7781 --- /dev/null +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.S @@ -0,0 +1,131 @@ +#include "rseq.h" + +#ifdef __x86_64__ + .text + .code64 + +#define FETCH_CPU(dest) movl %fs:__rseq_current_cpu@TPOFF, dest +#define CRITICAL_SECTION_OFFSET(label) $label + +/* If start <= %RESTART_ADDR_REG < %end, jump to jump_to */ +#define HANDLE_REGION(start, end, jump_to) \ + cmpqCRITICAL_SECTION_OFFSET(end), %RESTART_ADDR_REG; \ + jge 1f; \ + cmpqCRITICAL_SECTION_OFFSET(start), %RESTART_ADDR_REG; \ + jge jump_to; \ + 1:; + +#define HANDLE_REGION_PREFIX(prefix, start, end, jump_to) \ + HANDLE_REGION(prefix##start, prefix##end, prefix##jump_to) + +/*- + * Start of actual restartable sequences. + *---*/ + .align 8 + .globl RSEQ_CRITICAL_SECTION_START +RSEQ_CRITICAL_SECTION_START: +/* int rseq_percpu_lock() */ + .globl rseq_percpu_lock + .type rseq_percpu_lock, @function +rseq_percpu_lock: + .cfi_startproc +rseq_percpu_lock_region0: + FETCH_CPU(%eax) + leaq (,%eax,8), %RESTART_ADDR_REG + leaq (%rdi,%RESTART_ADDR_REG,8), %RESTART_ADDR_REG +rseq_percpu_lock_retry: + cmpw $0, (%RESTART_ADDR_REG) + jne rseq_percpu_lock_retry + movw $1, (%RESTART_ADDR_REG) /* 1 => lock owned */ +rseq_percpu_lock_region1: + ret +rseq_percpu_lock_region2: + .cfi_endproc + +/* + * int rseq_cmpxchg(int cpu, intptr_t *p, intptr_t old, intptr_t new) + * int rseq_percpu_cmpxchgcheck(int cpu, intptr_t *p, + * intptr_t old, intptr_t new, + * intptr_t *check_ptr, intptr_t check_val) + * + * NOTE: We don't use cmpxchg in the implementation below as that would make + * checking the success of our commit operation was dependent on flags (which + * are in turn clobbered by the restart region) -- furthermore we can't just + * retry to fill in the flags since the restarted cmpxchg may have actually + * succeeded; spuriously failing subsequent attempts. + */ + + .globl rseq_percpu_cmpxchg + .type rseq_percpu_cmpxchg, @function +rseq_percpu_cmpxchg: + .cfi_startproc +rseq_percpu_cmpxchg_region0: + FETCH_CPU(%eax) + cmp %eax, %edi /* check cpu vs current_cpu */ + jne rseq_percpu_cmpxchg_region1 + cmp %rdx, (%rsi) /* verify *p == old */ + jne rseq_percpu_cmpxchg_region2 + mov %rcx, (%rsi) +rseq_percpu_cmpxchg_region1: + ret/* return current cpu, indicating mismatch OR success */ +rseq_percpu_cmpxchg_region2: + mov $-1, %eax /* mismatch versus "old" or "check", return -1 */ + ret +rseq_percpu_cmpxchg_region3: + .cfi_endproc + + .globl rseq_percpu_cmpxchgcheck + .type rseq_percpu_cmpxchgcheck, @function +rseq_percpu_cmpxch
