This is an automated email from the ASF dual-hosted git repository. xiaoxiang781216 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nuttx.git
commit 840b46b1ad4f0a09309ece67a50e8bb4fcd1a622 Author: ouyangxiangzhen <[email protected]> AuthorDate: Mon Apr 27 15:42:53 2026 +0800 sched/hrtimer: Inline hrtimer_process and reduce irq overhead on UP Inline hrtimer_process into hrtimer.h and remove the standalone hrtimer_process.c, reducing IRQ overhead on UP by keeping the expired-timer processing close to the interrupt fast path. Assisted-By: mimo-v2.6-pro Signed-off-by: ouyangxiangzhen <[email protected]> --- sched/hrtimer/CMakeLists.txt | 10 +- sched/hrtimer/Make.defs | 2 +- sched/hrtimer/hrtimer.h | 208 ++++++++++++++++++++++++++++++------- sched/hrtimer/hrtimer_cancel.c | 4 +- sched/hrtimer/hrtimer_initialize.c | 4 +- sched/hrtimer/hrtimer_process.c | 153 --------------------------- sched/hrtimer/hrtimer_start.c | 4 +- 7 files changed, 177 insertions(+), 208 deletions(-) diff --git a/sched/hrtimer/CMakeLists.txt b/sched/hrtimer/CMakeLists.txt index 83328f3ce43..74020614a53 100644 --- a/sched/hrtimer/CMakeLists.txt +++ b/sched/hrtimer/CMakeLists.txt @@ -23,14 +23,8 @@ # Add hrtimer-related files to the build set(CSRCS) if(CONFIG_HRTIMER) - list( - APPEND - CSRCS - hrtimer_cancel.c - hrtimer_initialize.c - hrtimer_process.c - hrtimer_start.c - hrtimer_gettime.c) + list(APPEND CSRCS hrtimer_cancel.c hrtimer_initialize.c hrtimer_start.c + hrtimer_gettime.c) endif() target_sources(sched PRIVATE ${CSRCS}) diff --git a/sched/hrtimer/Make.defs b/sched/hrtimer/Make.defs index c95a9a4dc91..dfd326ad61b 100644 --- a/sched/hrtimer/Make.defs +++ b/sched/hrtimer/Make.defs @@ -23,7 +23,7 @@ # Add hrtimer-related files to the build ifeq ($(CONFIG_HRTIMER),y) - CSRCS += hrtimer_cancel.c hrtimer_initialize.c hrtimer_process.c hrtimer_start.c + CSRCS += hrtimer_cancel.c hrtimer_initialize.c hrtimer_start.c CSRCS += hrtimer_gettime.c endif diff --git a/sched/hrtimer/hrtimer.h b/sched/hrtimer/hrtimer.h index 52466a548d9..ba7d5828d39 100644 --- a/sched/hrtimer/hrtimer.h +++ b/sched/hrtimer/hrtimer.h @@ -94,40 +94,68 @@ extern struct FAR hrtimer_s *g_hrtimer_head; extern struct list_node g_hrtimer_list; #endif +#ifdef CONFIG_SMP /* Array of pointers to currently running high-resolution timers * for each CPU in SMP configurations. Index corresponds to CPU ID. */ -#ifdef CONFIG_SMP extern uintptr_t g_hrtimer_running[CONFIG_SMP_NCPUS]; #endif /**************************************************************************** - * Public Function Prototypes + * Inline Functions ****************************************************************************/ /**************************************************************************** - * Name: hrtimer_process + * Name: hrtimer_lock * * Description: - * Called from the timer interrupt handler to process expired - * high-resolution timers. If a timer has expired, its callback - * function will be executed in the context of the timer interrupt. + * Acquire the hrtimer queue lock. In SMP mode, this acquires the seqlock + * and disables interrupts. In non-SMP mode, only interrupts are disabled + * since no concurrent writers exist, saving the seqlock overhead. * * Input Parameters: - * now - The current time (nsecs). + * None. * * Returned Value: - * None. + * The saved interrupt state. * ****************************************************************************/ -void hrtimer_process(uint64_t now); +static inline_function irqstate_t hrtimer_lock(void) +{ +#ifdef CONFIG_SMP + return write_seqlock_irqsave(&g_hrtimer_lock); +#else + return up_irq_save(); +#endif +} /**************************************************************************** - * Inline Functions + * Name: hrtimer_unlock + * + * Description: + * Release the hrtimer queue lock. In SMP mode, this releases the seqlock + * and restores interrupts. In non-SMP mode, only interrupts are restored. + * + * Input Parameters: + * flags - The saved interrupt state from hrtimer_lock(). + * + * Returned Value: + * None. + * ****************************************************************************/ +static inline_function +void hrtimer_unlock(irqstate_t flags) +{ +#ifdef CONFIG_SMP + write_sequnlock_irqrestore(&g_hrtimer_lock, flags); +#else + up_irq_restore(flags); +#endif +} + /**************************************************************************** * Name: hrtimer_last_expired * @@ -371,46 +399,45 @@ static inline_function FAR hrtimer_t *hrtimer_get_first(void) * ****************************************************************************/ +#ifdef CONFIG_SMP +# define DEFINE_HRTIMER_READ(suffix, type) \ +static inline_function type \ +hrtimer_read_##suffix(FAR const type *ptr) \ +{ \ + type val; \ + uint32_t seq; \ + do \ + { \ + seq = read_seqbegin(&g_hrtimer_lock); \ + val = *ptr; \ + } \ + while (read_seqretry(&g_hrtimer_lock, seq)); \ + return val; \ +} +#else +# define DEFINE_HRTIMER_READ(suffix, type) \ +static inline_function type \ +hrtimer_read_##suffix(FAR const type *ptr) \ +{ \ + irqstate_t flags = up_irq_save(); \ + type val = *ptr; \ + up_irq_restore(flags); \ + return val; \ +} +#endif + #ifdef CONFIG_ARCH_64BIT /* On 64-bit architectures, read/write uint64_t is atomic. */ # define hrtimer_read_64(ptr) (*(FAR volatile uint64_t *)(ptr)) #else -static inline_function -uint64_t hrtimer_read_64(FAR const uint64_t *ptr) -{ - uint64_t val; - uint32_t seq; - - do - { - seq = read_seqbegin(&g_hrtimer_lock); - val = *ptr; - } - while (read_seqretry(&g_hrtimer_lock, seq)); - - return val; -} +DEFINE_HRTIMER_READ(64, uint64_t) #endif #if UINT_MAX >= UINT32_MAX /* On 32/64-bit architectures, read/write uint32_t is atomic. */ # define hrtimer_read_32(ptr) (*(FAR volatile uint32_t *)(ptr)) #else -static inline_function -uint32_t hrtimer_read_32(FAR const uint32_t *ptr) -{ - uint32_t val; - uint32_t seq; - - do - { - seq = read_seqbegin(&g_hrtimer_lock); - val = *ptr; - } - while (read_seqretry(&g_hrtimer_lock, seq)); - - return val; -} +DEFINE_HRTIMER_READ(32, uint32_t) #endif /* Generic function to read the pointer in the queue atomically. */ @@ -584,5 +611,106 @@ static inline_function void hrtimer_wait(FAR hrtimer_t *timer) # define hrtimer_wait(timer) /* Nothing to wait in non-SMP. */ #endif +/**************************************************************************** + * Name: hrtimer_process + * + * Description: + * Called from the timer interrupt handler to process expired + * high-resolution timers. If a timer has expired, its callback + * function will be executed in the context of the timer interrupt. + * + * Input Parameters: + * now - The current time (nsecs). + * + * Returned Value: + * None. + * + * Assumption: + * This function can only be called in interrupt context. + * + ****************************************************************************/ + +static inline_function void hrtimer_process(uint64_t now) +{ + FAR hrtimer_t *hrtimer; + hrtimer_entry_t func; + uint64_t expired; + uint64_t delay; + int cpu = this_cpu(); + irqstate_t flags = hrtimer_lock(); + + /* Acquire the lock and seize the ownership of the hrtimer queue. */ + + for (; ; ) + { + /* Fetch the earliest active timer */ + + hrtimer = hrtimer_get_first(); + expired = hrtimer->expired; + + /* Check if the timer has expired */ + + if (!HRTIMER_TIME_BEFORE_EQ(expired, now)) + { + break; + } + + /* Remove the expired timer from the timer queue */ + + func = hrtimer->func; + hrtimer_remove(hrtimer); + + hrtimer_mark_running(hrtimer, cpu); + + /* Leave critical section before invoking the callback */ + + hrtimer_unlock(flags); + + /* Invoke the timer callback */ + + delay = func(hrtimer, expired); + + /* Ensure the delay is valid. */ + + DEBUGASSERT(HRTIMER_TIME_BEFORE_EQ(expired + delay, + now + HRTIMER_MAX_DELAY)); + + /* Re-enter critical section to update timer state */ + + flags = hrtimer_lock(); + + /* If the timer is periodic and has not been rearmed or + * cancelled concurrently, calculate next expiration and + * re-insert into the timer queue. + */ + + if (delay != 0u && hrtimer_is_running(hrtimer, cpu)) + { + hrtimer->expired = expired + delay; + hrtimer->func = func; + hrtimer_insert(hrtimer); + } + } + + hrtimer_unmark_running(cpu); + + /* Note that the g_hrtimer_guard.expired is not monotonic. */ + + g_hrtimer_guard.expired = now + HRTIMER_GUARD_DELAY; + + /* Schedule the next timer expiration */ + + if (expired != now) + { + /* Start timer for the next earliest expiration */ + + hrtimer_reprogram(expired); + } + + /* Release the lock and give up the ownership of the hrtimer queue. */ + + hrtimer_unlock(flags); +} + #endif /* CONFIG_HRTIMER */ #endif /* __SCHED_HRTIMER_HRTIMER_H */ diff --git a/sched/hrtimer/hrtimer_cancel.c b/sched/hrtimer/hrtimer_cancel.c index 50de9e15981..fecd0a16821 100644 --- a/sched/hrtimer/hrtimer_cancel.c +++ b/sched/hrtimer/hrtimer_cancel.c @@ -82,7 +82,7 @@ int hrtimer_cancel(FAR hrtimer_t *hrtimer) /* Acquire the lock and seize the ownership of the hrtimer queue. */ - flags = write_seqlock_irqsave(&g_hrtimer_lock); + flags = hrtimer_lock(); /* Ensure no core can write the hrtimer. */ @@ -101,7 +101,7 @@ int hrtimer_cancel(FAR hrtimer_t *hrtimer) /* Release the lock and give up the ownership of the hrtimer queue. */ - write_sequnlock_irqrestore(&g_hrtimer_lock, flags); + hrtimer_unlock(flags); return ret; } diff --git a/sched/hrtimer/hrtimer_initialize.c b/sched/hrtimer/hrtimer_initialize.c index f5cbba6c4a5..0b8beaf53cb 100644 --- a/sched/hrtimer/hrtimer_initialize.c +++ b/sched/hrtimer/hrtimer_initialize.c @@ -32,13 +32,12 @@ * Public Data ****************************************************************************/ +#ifdef CONFIG_SMP /* Array of pointers to currently running high-resolution timers * for each CPU in SMP configurations. Index corresponds to CPU ID. */ -#ifdef CONFIG_SMP uintptr_t g_hrtimer_running[CONFIG_SMP_NCPUS]; -#endif /* Global spinlock protecting the high-resolution timer subsystem. * @@ -48,6 +47,7 @@ uintptr_t g_hrtimer_running[CONFIG_SMP_NCPUS]; */ seqcount_t g_hrtimer_lock = SEQLOCK_INITIALIZER; +#endif /* CONFIG_SMP */ /* HRTimer queue for all active high-resolution timers. * diff --git a/sched/hrtimer/hrtimer_process.c b/sched/hrtimer/hrtimer_process.c deleted file mode 100644 index 6d295be8ea2..00000000000 --- a/sched/hrtimer/hrtimer_process.c +++ /dev/null @@ -1,153 +0,0 @@ -/**************************************************************************** - * sched/hrtimer/hrtimer_process.c - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - ****************************************************************************/ - -/**************************************************************************** - * Included Files - ****************************************************************************/ - -#include <nuttx/config.h> -#include <assert.h> -#include <nuttx/arch.h> -#include <nuttx/clock.h> - -#include "hrtimer/hrtimer.h" - -/**************************************************************************** - * Public Functions - ****************************************************************************/ - -/**************************************************************************** - * Name: hrtimer_process - * - * Description: - * Process all expired high-resolution timers. This function repeatedly - * retrieves the earliest timer from the active timer queue, checks - * if it has expired relative to the current time, removes it from the - * queue, and invokes its callback function. Processing continues - * until: - * - * 1. No additional timers have expired, or - * 2. The active timer set is empty. - * - * After all expired timers are processed, the next expiration event is - * scheduled based on: - * - * - The earliest remaining timer, or - * - A fallback expiration (current time + HRTIMER_DEFAULT_INCREMENT) - * if no timers remain. - * - * Input Parameters: - * now - Current high-resolution timestamp. - * - * Returned Value: - * None. - * - * Assumptions/Notes: - * - This function acquires a spinlock to protect the timer queue. - * - Timer callbacks are invoked with interrupts enabled - * to avoid deadlocks. - * - DEBUGASSERT ensures that timer callbacks are valid. - ****************************************************************************/ - -void hrtimer_process(uint64_t now) -{ - FAR hrtimer_t *hrtimer; - irqstate_t flags; - hrtimer_entry_t func; - uint64_t expired; - uint64_t delay; - int cpu = this_cpu(); - - /* Acquire the lock and seize the ownership of the hrtimer queue. */ - - flags = write_seqlock_irqsave(&g_hrtimer_lock); - - for (; ; ) - { - /* Fetch the earliest active timer */ - - hrtimer = hrtimer_get_first(); - expired = hrtimer->expired; - - /* Check if the timer has expired */ - - if (!HRTIMER_TIME_BEFORE_EQ(expired, now)) - { - break; - } - - /* Remove the expired timer from the timer queue */ - - func = hrtimer->func; - hrtimer_remove(hrtimer); - - hrtimer_mark_running(hrtimer, cpu); - - /* Leave critical section before invoking the callback */ - - write_sequnlock_irqrestore(&g_hrtimer_lock, flags); - - /* Invoke the timer callback */ - - delay = func(hrtimer, expired); - - /* Ensure the delay is valid. */ - - DEBUGASSERT(HRTIMER_TIME_BEFORE_EQ(expired + delay, - now + HRTIMER_MAX_DELAY)); - - /* Re-enter critical section to update timer state */ - - flags = write_seqlock_irqsave(&g_hrtimer_lock); - - /* If the timer is periodic and has not been rearmed or - * cancelled concurrently, calculate next expiration and - * re-insert into the timer queue. - */ - - if (delay != 0u && hrtimer_is_running(hrtimer, cpu)) - { - hrtimer->expired = expired + delay; - hrtimer->func = func; - hrtimer_insert(hrtimer); - } - } - - hrtimer_unmark_running(cpu); - - /* Note that the g_hrtimer_guard.expired is not monotonic. */ - - g_hrtimer_guard.expired = now + HRTIMER_GUARD_DELAY; - - /* Schedule the next timer expiration */ - - if (expired != now) - { - /* Start timer for the next earliest expiration */ - - hrtimer_reprogram(expired); - } - - /* Release the lock and give up the ownership of the hrtimer queue. */ - - write_sequnlock_irqrestore(&g_hrtimer_lock, flags); -} diff --git a/sched/hrtimer/hrtimer_start.c b/sched/hrtimer/hrtimer_start.c index 23c817abd70..4ec4e0ca6c7 100644 --- a/sched/hrtimer/hrtimer_start.c +++ b/sched/hrtimer/hrtimer_start.c @@ -66,7 +66,7 @@ int hrtimer_start_absolute(FAR hrtimer_t *hrtimer, hrtimer_entry_t func, /* Acquire the lock and seize the ownership of the hrtimer queue. */ - flags = write_seqlock_irqsave(&g_hrtimer_lock); + flags = hrtimer_lock(); /* Ensure no running core can write the hrtimer. */ @@ -93,7 +93,7 @@ int hrtimer_start_absolute(FAR hrtimer_t *hrtimer, hrtimer_entry_t func, /* Release the lock and give up the ownership of the hrtimer queue. */ - write_sequnlock_irqrestore(&g_hrtimer_lock, flags); + hrtimer_unlock(flags); return ret; }
