[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

3.14.79-rt85-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 9afdfb9ce021..aba26ff51f39 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -107,6 +107,8 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -119,8 +121,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(__this_cpu_read_4(__preempt_count) == preempt_offset || 
\
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = __this_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(__this_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

3.14.79-rt85-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 9afdfb9ce021..aba26ff51f39 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -107,6 +107,8 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -119,8 +121,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(__this_cpu_read_4(__preempt_count) == preempt_offset || 
\
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = __this_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(__this_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

3.18.42-rt45-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 16 ++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 060d1b4e475d..6806369bddf5 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -95,6 +95,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -107,8 +109,18 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(void)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(!raw_cpu_read_4(__preempt_count) || \
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   if (!raw_cpu_read_4(__preempt_count))
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(!raw_cpu_read_4(__preempt_count));
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

3.18.42-rt45-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 16 ++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 060d1b4e475d..6806369bddf5 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -95,6 +95,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -107,8 +109,18 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(void)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(!raw_cpu_read_4(__preempt_count) || \
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   if (!raw_cpu_read_4(__preempt_count))
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(!raw_cpu_read_4(__preempt_count));
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

4.1.33-rt38-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index c08949b0314d..eff1b8609f77 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -92,6 +92,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -104,8 +106,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset ||
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = raw_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

4.1.33-rt38-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index c08949b0314d..eff1b8609f77 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -92,6 +92,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -104,8 +106,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset ||
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = raw_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

4.4.21-rt31-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 5dbd2d0f91e0..6f432adc55cd 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -89,6 +89,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -101,8 +103,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset ||
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = raw_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1

[PATCH RT 08/10] x86/preempt-lazy: fixup should_resched()

[Steven Rostedt]

4.4.21-rt31-rc1 stable review patch.
If anyone has any objections, please let me know.

--

From: Sebastian Andrzej Siewior 

should_resched() returns true if NEED_RESCHED is set and the
preempt_count is 0 _or_ if NEED_RESCHED_LAZY is set ignoring the preempt
counter. Ignoring the preemp counter is wrong. This patch adds this into
account.
While at it, __preempt_count_dec_and_test() ignores preempt_lazy_count
while checking TIF_NEED_RESCHED_LAZY so we this check, too.

Signed-off-by: Sebastian Andrzej Siewior 
Signed-off-by: Steven Rostedt 
---
 arch/x86/include/asm/preempt.h | 17 +++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 5dbd2d0f91e0..6f432adc55cd 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -89,6 +89,8 @@ static __always_inline bool __preempt_count_dec_and_test(void)
if (preempt_count_dec_and_test())
return true;
 #ifdef CONFIG_PREEMPT_LAZY
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return false;
@@ -101,8 +103,19 @@ static __always_inline bool 
__preempt_count_dec_and_test(void)
 static __always_inline bool should_resched(int preempt_offset)
 {
 #ifdef CONFIG_PREEMPT_LAZY
-   return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset ||
-   test_thread_flag(TIF_NEED_RESCHED_LAZY));
+   u32 tmp;
+
+   tmp = raw_cpu_read_4(__preempt_count);
+   if (tmp == preempt_offset)
+   return true;
+
+   /* preempt count == 0 ? */
+   tmp &= ~PREEMPT_NEED_RESCHED;
+   if (tmp)
+   return false;
+   if (current_thread_info()->preempt_lazy_count)
+   return false;
+   return test_thread_flag(TIF_NEED_RESCHED_LAZY);
 #else
return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
 #endif
-- 
2.8.1