Re: [PATCH] wireguard: convert selftest/{counter,ratelimiter}.c to KUnit
Hi Daniel, Thank you for the patch! Perhaps something to improve: [auto build test WARNING on 7cf726a59435301046250c42131554d9ccc566b8] url: https://github.com/0day-ci/linux/commits/Daniel-Latypov/wireguard-convert-selftest-counter-ratelimiter-c-to-KUnit/20201020-042650 base:7cf726a59435301046250c42131554d9ccc566b8 config: mips-allyesconfig (attached as .config) compiler: mips-linux-gcc (GCC) 9.3.0 reproduce (this is a W=1 build): wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross chmod +x ~/bin/make.cross # https://github.com/0day-ci/linux/commit/7a0f82af0af9735a7f20ef9e291e704aff218e8f git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Daniel-Latypov/wireguard-convert-selftest-counter-ratelimiter-c-to-KUnit/20201020-042650 git checkout 7a0f82af0af9735a7f20ef9e291e704aff218e8f # save the attached .config to linux build tree COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=mips If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot All warnings (new ones prefixed by >>): drivers/net/wireguard/counter_test.c:84:2: note: in expansion of macro 'T' 84 | T(COUNTER_WINDOW_SIZE + 1, true); | ^ include/linux/minmax.h:18:28: warning: comparison of distinct pointer types lacks a cast 18 | (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) |^~ include/kunit/test.h:748:9: note: in expansion of macro '__typecheck' 748 | ((void)__typecheck(__left, __right)); \ | ^~~ include/kunit/test.h:772:2: note: in expansion of macro 'KUNIT_BASE_BINARY_ASSERTION' 772 | KUNIT_BASE_BINARY_ASSERTION(test, \ | ^~~ include/kunit/test.h:861:2: note: in expansion of macro 'KUNIT_BASE_EQ_MSG_ASSERTION' 861 | KUNIT_BASE_EQ_MSG_ASSERTION(test, \ | ^~~ include/kunit/test.h:871:2: note: in expansion of macro 'KUNIT_BINARY_EQ_MSG_ASSERTION' 871 | KUNIT_BINARY_EQ_MSG_ASSERTION(test, \ | ^ include/kunit/test.h:1234:2: note: in expansion of macro 'KUNIT_BINARY_EQ_ASSERTION' 1234 | KUNIT_BINARY_EQ_ASSERTION(test, KUNIT_EXPECTATION, left, right) | ^ drivers/net/wireguard/counter_test.c:22:3: note: in expansion of macro 'KUNIT_EXPECT_EQ' 22 | KUNIT_EXPECT_EQ(test, counter_validate(counter, n), v) | ^~~ drivers/net/wireguard/counter_test.c:85:2: note: in expansion of macro 'T' 85 | T(0, false); | ^ include/linux/minmax.h:18:28: warning: comparison of distinct pointer types lacks a cast 18 | (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) |^~ include/kunit/test.h:748:9: note: in expansion of macro '__typecheck' 748 | ((void)__typecheck(__left, __right)); \ | ^~~ include/kunit/test.h:772:2: note: in expansion of macro 'KUNIT_BASE_BINARY_ASSERTION' 772 | KUNIT_BASE_BINARY_ASSERTION(test, \ | ^~~ include/kunit/test.h:861:2: note: in expansion of macro 'KUNIT_BASE_EQ_MSG_ASSERTION' 861 | KUNIT_BASE_EQ_MSG_ASSERTION(test, \ | ^~~ include/kunit/test.h:871:2: note: in expansion of macro 'KUNIT_BINARY_EQ_MSG_ASSERTION' 871 | KUNIT_BINARY_EQ_MSG_ASSERTION(test, \ | ^ include/kunit/test.h:1234:2: note: in expansion of macro 'KUNIT_BINARY_EQ_ASSERTION' 1234 | KUNIT_BINARY_EQ_ASSERTION(test, KUNIT_EXPECTATION, left, right) | ^ drivers/net/wireguard/counter_test.c:22:3: note: in expansion of macro 'KUNIT_EXPECT_EQ' 22 | KUNIT_EXPECT_EQ(test, counter_validate(counter, n), v) | ^~~ drivers/net/wireguard/counter_test.c:89:3: note: in expansion of macro 'T' 89 | T(i, true); | ^ include/linux/minmax.h:18:28: warning: comparison of distinct pointer types lacks a cast 18 | (!!(sizeof((typeof(x) *)1 == (typeof(y) *)1))) |^~ include/kunit/test.h:748:9: note: in expansion of macro '__typecheck' 748 | ((void)__typecheck(__left, __right)); \ | ^~~ include/kunit/test.h:772:2: note: in expansion of macro 'KUNIT_BASE_BINARY_ASSERTION' 772 | KUNIT_BASE_BINARY_ASSERTION(test, \ | ^~~ include/kunit/test.h:861:2: note: in expansion of macro 'KUNIT_BASE_EQ_MSG_ASSERTION' 861 | KUNIT_BASE_EQ_MSG_ASSERTION(test, \ | ^~~
Re: [PATCH] wireguard: convert selftest/{counter,ratelimiter}.c to KUnit
Hi Daniel, Thank you for the patch! Yet something to improve: [auto build test ERROR on 7cf726a59435301046250c42131554d9ccc566b8] url: https://github.com/0day-ci/linux/commits/Daniel-Latypov/wireguard-convert-selftest-counter-ratelimiter-c-to-KUnit/20201020-042650 base:7cf726a59435301046250c42131554d9ccc566b8 config: x86_64-allmodconfig (attached as .config) compiler: gcc-9 (Debian 9.3.0-15) 9.3.0 reproduce (this is a W=1 build): # https://github.com/0day-ci/linux/commit/7a0f82af0af9735a7f20ef9e291e704aff218e8f git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Daniel-Latypov/wireguard-convert-selftest-counter-ratelimiter-c-to-KUnit/20201020-042650 git checkout 7a0f82af0af9735a7f20ef9e291e704aff218e8f # save the attached .config to linux build tree make W=1 ARCH=x86_64 If you fix the issue, kindly add following tag as appropriate Reported-by: kernel test robot All errors (new ones prefixed by >>): ld: drivers/net/wireguard/receive.o: in function `kunit_test_suites_init': >> receive.c:(.init.text+0x0): multiple definition of `init_module'; >> drivers/net/wireguard/main.o:main.c:(.init.text+0x0): first defined here ld: drivers/net/wireguard/receive.o: in function `kunit_test_suites_exit': >> receive.c:(.exit.text+0x0): multiple definition of `cleanup_module'; >> drivers/net/wireguard/main.o:main.c:(.exit.text+0x0): first defined here ld: drivers/net/wireguard/ratelimiter.o: in function `kunit_test_suites_init': ratelimiter.c:(.init.text+0x0): multiple definition of `init_module'; drivers/net/wireguard/main.o:main.c:(.init.text+0x0): first defined here ld: drivers/net/wireguard/ratelimiter.o: in function `kunit_test_suites_exit': ratelimiter.c:(.exit.text+0x0): multiple definition of `cleanup_module'; drivers/net/wireguard/main.o:main.c:(.exit.text+0x0): first defined here --- 0-DAY CI Kernel Test Service, Intel Corporation https://lists.01.org/hyperkitty/list/kbuild-...@lists.01.org .config.gz Description: application/gzip
Re: [PATCH] wireguard: convert selftest/{counter,ratelimiter}.c to KUnit
On Mon, Oct 19, 2020 at 3:36 PM Jason A. Donenfeld wrote:
>
> Hi Daniel,
>
> Thanks for this patch. KUnit looks interesting. But I'm not totally
> sure what this would gain here, except more complicated infrastructure
> to handle. We're already running these tests in our CI on every single
> commit made to a variety of trees on a variety of architectures -- see
> https://www.wireguard.com/build-status/ -- runnable via `make -C
> tools/testing/selftests/wireguard/qemu -j$(nproc)`. It looks like this
> commit breaks that while making everything slightly more complex. Is
Thanks for the informative response and the pointer to the
build-status/ page, I was unaware of its existence.
Digging into that a bit deeper, I'd agree with you that this patch
isn't worth it.
Yes, this would stop these two tests from running under selftests, and
thus in your CI.
A not-too-long glance at the code made it seem like the specific
code-under-test here was reasonably arch-independent, but yes, this
would make it more annoying to test different arches.
> there a good reason to switch over to this other than fad? From a
> development perspective, I don't see this as really helping with much.
In my mind, the breakdown is
Pros:
* more minimal environment
* config file is 6 lines, instead of 87
* doesn't rely on a userspace or a custom init, etc.
* slightly faster build times on my machine (with -j8)
* the option to provide a bit more structure via its MACROS
* but that's optional, can fall back to `if (success) KUNIT_FAIL("my
message")`
Cons:
* separate set of tooling needed to run tests
* needs to be then integrated into WireGuard's CI
* not as mature, so it lacks integration via KernelCI, etc.
* Brendan (CC'd) is working on this KernelCI integration in particular.
* qemu/init.c has more features that KUnit currently lacks, like kmemleak checks
* and others I'm not able to think of.
WG's tooling is really nice, so these cons are much more apparent.
I think a feature that might make this worth looking at again later on
is if selftest modules could be written using KUnit.
Commit c475c77d5b56 ("kunit: allow kunit tests to be loaded as a
module") was necessary but not sufficient here.
If that becomes possible, KUnit would mainly provide the boilerplate
for tracking pass-fail, generating error messages, and the *option* of
running the tests via either KUnit or Kselftest.
Until that time, I'd agree that WG is better off as-is.
Cheers,
Daniel
>
> Jason
>
> On Mon, Oct 19, 2020 at 10:24 PM Daniel Latypov wrote:
> >
> > These tests already focus on testing individual functions that can run
> > in a more minimal environment like KUnit.
> >
> > The primary motivation for this change it to make it faster and easier
> > to run these tests, and thus encourage the addition of more test cases.
> >
> > E.g.
> > Test timing after make mrproper: 47.418s building, 0.000s running
> > With an incremental build: 3.891s building, 0.000s running
> >
> > KUnit also provides a bit more structure, like tracking overall
> > pass/fail status and printing failure messages like
> > > # wg_packet_counter_test: EXPECTATION FAILED at
> > > drivers/net/wireguard/counter_test.c:32
> > > Expected counter_validate(counter, (COUNTER_WINDOW_SIZE + 1)) == false,
> > > but
> >
> > Note: so we no longer need to track test_num in counter_test.c.
> > But deleting the /*1*/ test_num comments means git (with the default
> > threshold) no longer recognizes that the file was moved.
> >
> > Signed-off-by: Daniel Latypov
> > Cc: Jason A. Donenfeld
> > Cc: David Miller
> > Cc: Brendan Higgins
> > ---
> > drivers/net/Kconfig | 12
> > .../{selftest/counter.c => counter_test.c}| 45 ++--
> > drivers/net/wireguard/main.c | 3 +-
> > drivers/net/wireguard/queueing.h | 4 --
> > drivers/net/wireguard/ratelimiter.c | 4 +-
> > .../ratelimiter.c => ratelimiter_test.c} | 68 +++
> > drivers/net/wireguard/receive.c | 6 +-
> > 7 files changed, 80 insertions(+), 62 deletions(-)
> > rename drivers/net/wireguard/{selftest/counter.c => counter_test.c} (73%)
> > rename drivers/net/wireguard/{selftest/ratelimiter.c =>
> > ratelimiter_test.c} (85%)
> >
> > diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
> > index c3dbe64e628e..208ed162bcc0 100644
> > --- a/drivers/net/Kconfig
> > +++ b/drivers/net/Kconfig
> > @@ -114,6 +114,18 @@ config WIREGUARD_DEBUG
> >
> > Say N here unless you know what you're doing.
> >
> > +config WIREGUARD_KUNIT_TEST
> > + tristate "KUnit tests for WireGuard"
> > + default KUNIT_ALL_TESTS
> > + depends on KUNIT && WIREGUARD
> > + help
> > + This enables KUnit tests for Wireguard.
> > +
> > + For more information on KUnit and unit tests in general please
> > refer
> > + to the KUnit documentation in Documentation/dev-tools/kunit/.
> > +
> > + Say N
Re: [PATCH] wireguard: convert selftest/{counter,ratelimiter}.c to KUnit
Hi Daniel,
Thanks for this patch. KUnit looks interesting. But I'm not totally
sure what this would gain here, except more complicated infrastructure
to handle. We're already running these tests in our CI on every single
commit made to a variety of trees on a variety of architectures -- see
https://www.wireguard.com/build-status/ -- runnable via `make -C
tools/testing/selftests/wireguard/qemu -j$(nproc)`. It looks like this
commit breaks that while making everything slightly more complex. Is
there a good reason to switch over to this other than fad? From a
development perspective, I don't see this as really helping with much.
Jason
On Mon, Oct 19, 2020 at 10:24 PM Daniel Latypov wrote:
>
> These tests already focus on testing individual functions that can run
> in a more minimal environment like KUnit.
>
> The primary motivation for this change it to make it faster and easier
> to run these tests, and thus encourage the addition of more test cases.
>
> E.g.
> Test timing after make mrproper: 47.418s building, 0.000s running
> With an incremental build: 3.891s building, 0.000s running
>
> KUnit also provides a bit more structure, like tracking overall
> pass/fail status and printing failure messages like
> > # wg_packet_counter_test: EXPECTATION FAILED at
> > drivers/net/wireguard/counter_test.c:32
> > Expected counter_validate(counter, (COUNTER_WINDOW_SIZE + 1)) == false, but
>
> Note: so we no longer need to track test_num in counter_test.c.
> But deleting the /*1*/ test_num comments means git (with the default
> threshold) no longer recognizes that the file was moved.
>
> Signed-off-by: Daniel Latypov
> Cc: Jason A. Donenfeld
> Cc: David Miller
> Cc: Brendan Higgins
> ---
> drivers/net/Kconfig | 12
> .../{selftest/counter.c => counter_test.c}| 45 ++--
> drivers/net/wireguard/main.c | 3 +-
> drivers/net/wireguard/queueing.h | 4 --
> drivers/net/wireguard/ratelimiter.c | 4 +-
> .../ratelimiter.c => ratelimiter_test.c} | 68 +++
> drivers/net/wireguard/receive.c | 6 +-
> 7 files changed, 80 insertions(+), 62 deletions(-)
> rename drivers/net/wireguard/{selftest/counter.c => counter_test.c} (73%)
> rename drivers/net/wireguard/{selftest/ratelimiter.c => ratelimiter_test.c}
> (85%)
>
> diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
> index c3dbe64e628e..208ed162bcc0 100644
> --- a/drivers/net/Kconfig
> +++ b/drivers/net/Kconfig
> @@ -114,6 +114,18 @@ config WIREGUARD_DEBUG
>
> Say N here unless you know what you're doing.
>
> +config WIREGUARD_KUNIT_TEST
> + tristate "KUnit tests for WireGuard"
> + default KUNIT_ALL_TESTS
> + depends on KUNIT && WIREGUARD
> + help
> + This enables KUnit tests for Wireguard.
> +
> + For more information on KUnit and unit tests in general please refer
> + to the KUnit documentation in Documentation/dev-tools/kunit/.
> +
> + Say N here unless you know what you're doing.
> +
> config EQUALIZER
> tristate "EQL (serial line load balancing) support"
> help
> diff --git a/drivers/net/wireguard/selftest/counter.c
> b/drivers/net/wireguard/counter_test.c
> similarity index 73%
> rename from drivers/net/wireguard/selftest/counter.c
> rename to drivers/net/wireguard/counter_test.c
> index ec3c156bf91b..167153fc249f 100644
> --- a/drivers/net/wireguard/selftest/counter.c
> +++ b/drivers/net/wireguard/counter_test.c
> @@ -3,32 +3,23 @@
> * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights
> Reserved.
> */
>
> -#ifdef DEBUG
> -bool __init wg_packet_counter_selftest(void)
> +#include
> +
> +static void wg_packet_counter_test(struct kunit *test)
> {
> struct noise_replay_counter *counter;
> - unsigned int test_num = 0, i;
> - bool success = true;
> + unsigned int i;
>
> - counter = kmalloc(sizeof(*counter), GFP_KERNEL);
> - if (unlikely(!counter)) {
> - pr_err("nonce counter self-test malloc: FAIL\n");
> - return false;
> - }
> + counter = kunit_kmalloc(test, sizeof(*counter), GFP_KERNEL);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, counter);
>
> #define T_INIT do {\
> memset(counter, 0, sizeof(*counter)); \
> spin_lock_init(>lock);\
> } while (0)
> #define T_LIM (COUNTER_WINDOW_SIZE + 1)
> -#define T(n, v) do { \
> - ++test_num; \
> - if (counter_validate(counter, n) != (v)) {\
> - pr_err("nonce counter self-test %u: FAIL\n", \
> - test_num); \
> - success = false; \
> - }
[PATCH] wireguard: convert selftest/{counter,ratelimiter}.c to KUnit
These tests already focus on testing individual functions that can run
in a more minimal environment like KUnit.
The primary motivation for this change it to make it faster and easier
to run these tests, and thus encourage the addition of more test cases.
E.g.
Test timing after make mrproper: 47.418s building, 0.000s running
With an incremental build: 3.891s building, 0.000s running
KUnit also provides a bit more structure, like tracking overall
pass/fail status and printing failure messages like
> # wg_packet_counter_test: EXPECTATION FAILED at
> drivers/net/wireguard/counter_test.c:32
> Expected counter_validate(counter, (COUNTER_WINDOW_SIZE + 1)) == false, but
Note: so we no longer need to track test_num in counter_test.c.
But deleting the /*1*/ test_num comments means git (with the default
threshold) no longer recognizes that the file was moved.
Signed-off-by: Daniel Latypov
Cc: Jason A. Donenfeld
Cc: David Miller
Cc: Brendan Higgins
---
drivers/net/Kconfig | 12
.../{selftest/counter.c => counter_test.c}| 45 ++--
drivers/net/wireguard/main.c | 3 +-
drivers/net/wireguard/queueing.h | 4 --
drivers/net/wireguard/ratelimiter.c | 4 +-
.../ratelimiter.c => ratelimiter_test.c} | 68 +++
drivers/net/wireguard/receive.c | 6 +-
7 files changed, 80 insertions(+), 62 deletions(-)
rename drivers/net/wireguard/{selftest/counter.c => counter_test.c} (73%)
rename drivers/net/wireguard/{selftest/ratelimiter.c => ratelimiter_test.c}
(85%)
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index c3dbe64e628e..208ed162bcc0 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -114,6 +114,18 @@ config WIREGUARD_DEBUG
Say N here unless you know what you're doing.
+config WIREGUARD_KUNIT_TEST
+ tristate "KUnit tests for WireGuard"
+ default KUNIT_ALL_TESTS
+ depends on KUNIT && WIREGUARD
+ help
+ This enables KUnit tests for Wireguard.
+
+ For more information on KUnit and unit tests in general please refer
+ to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+ Say N here unless you know what you're doing.
+
config EQUALIZER
tristate "EQL (serial line load balancing) support"
help
diff --git a/drivers/net/wireguard/selftest/counter.c
b/drivers/net/wireguard/counter_test.c
similarity index 73%
rename from drivers/net/wireguard/selftest/counter.c
rename to drivers/net/wireguard/counter_test.c
index ec3c156bf91b..167153fc249f 100644
--- a/drivers/net/wireguard/selftest/counter.c
+++ b/drivers/net/wireguard/counter_test.c
@@ -3,32 +3,23 @@
* Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights
Reserved.
*/
-#ifdef DEBUG
-bool __init wg_packet_counter_selftest(void)
+#include
+
+static void wg_packet_counter_test(struct kunit *test)
{
struct noise_replay_counter *counter;
- unsigned int test_num = 0, i;
- bool success = true;
+ unsigned int i;
- counter = kmalloc(sizeof(*counter), GFP_KERNEL);
- if (unlikely(!counter)) {
- pr_err("nonce counter self-test malloc: FAIL\n");
- return false;
- }
+ counter = kunit_kmalloc(test, sizeof(*counter), GFP_KERNEL);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, counter);
#define T_INIT do {\
memset(counter, 0, sizeof(*counter)); \
spin_lock_init(>lock);\
} while (0)
#define T_LIM (COUNTER_WINDOW_SIZE + 1)
-#define T(n, v) do { \
- ++test_num; \
- if (counter_validate(counter, n) != (v)) {\
- pr_err("nonce counter self-test %u: FAIL\n", \
- test_num); \
- success = false; \
- } \
- } while (0)
+#define T(n, v) \
+ KUNIT_EXPECT_EQ(test, counter_validate(counter, n), v)
T_INIT;
/* 1 */ T(0, true);
@@ -102,10 +93,18 @@ bool __init wg_packet_counter_selftest(void)
#undef T
#undef T_LIM
#undef T_INIT
-
- if (success)
- pr_info("nonce counter self-tests: pass\n");
- kfree(counter);
- return success;
}
-#endif
+
+static struct kunit_case wg_packet_counter_test_cases[] = {
+ KUNIT_CASE(wg_packet_counter_test),
+ {}
+};
+
+static struct kunit_suite wg_packet_counter_test_suite = {
+ .name = "wg_packet_counter",
+ .test_cases = wg_packet_counter_test_cases,
+};
+
+kunit_test_suites(_packet_counter_test_suite);
+
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/wireguard/main.c b/drivers/net/wireguard/main.c
index 7a7d5f1a80fc..bfd3312d5133 100644
---
