--enable-polled-timers runs timer expiration processing directly inside
odp_schedule() and avoids the use of timer pool background threads and
itimers.

odp_timers stress test shows improvements for test cases with higher resolution
and lower latency timers. The most noticable improvement is in scenarios where
the number of timer pools is greater than the number of control plane cores.

For further explanation please see:
https://docs.google.com/a/linaro.org/document/d/1AI0TFlIb3QJFAd3mJz74kPzMrLmiQgwmuOx31LlffZA/edit?usp=sharing

Signed-off-by: Brian Brooks <brian.bro...@linaro.org>
---
 configure.ac                                       | 13 +++++
 .../linux-generic/include/odp_config_internal.h    |  3 +
 platform/linux-generic/include/odp_time_internal.h |  2 +
 .../linux-generic/include/odp_timer_internal.h     |  2 +
 platform/linux-generic/odp_schedule.c              | 40 +++++++------
 platform/linux-generic/odp_time.c                  | 60 +++++++++++++++++++-
 platform/linux-generic/odp_timer.c                 | 65 +++++++++++++++++++++-
 7 files changed, 165 insertions(+), 20 deletions(-)

diff --git a/configure.ac b/configure.ac
index 4f6cc18..4f0d276 100644
--- a/configure.ac
+++ b/configure.ac
@@ -232,6 +232,19 @@ AC_ARG_ENABLE([tracing-timers],
 ODP_CFLAGS="$ODP_CFLAGS -DTRACING_TIMERS=$TRACING_TIMERS"
 
 ##########################################################################
+# Enable/disable ODP_POLLED_TIMERS
+##########################################################################
+ODP_POLLED_TIMERS=0
+AC_ARG_ENABLE([polled-timers],
+    [  --enable-polled-timers    polled timers],
+    [if test "x$enableval" = "xyes"; then
+       ODP_POLLED_TIMERS=1
+     else
+       ODP_POLLED_TIMERS=0
+    fi])
+ODP_CFLAGS="$ODP_CFLAGS -DODP_POLLED_TIMERS=$ODP_POLLED_TIMERS"
+
+##########################################################################
 # Enable/disable ODP_DEBUG_PRINT
 ##########################################################################
 ODP_DEBUG_PRINT=0
diff --git a/platform/linux-generic/include/odp_config_internal.h 
b/platform/linux-generic/include/odp_config_internal.h
index b7ff610..eec9dac 100644
--- a/platform/linux-generic/include/odp_config_internal.h
+++ b/platform/linux-generic/include/odp_config_internal.h
@@ -118,6 +118,9 @@ extern "C" {
  */
 #define CONFIG_BURST_SIZE 16
 
+/* Value used to rate limit timer pool expiration processing. */
+#define ODP_CONFIG_TIMER_RUN_NSEC  (250)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/platform/linux-generic/include/odp_time_internal.h 
b/platform/linux-generic/include/odp_time_internal.h
index 5a0bc75..1185f58 100644
--- a/platform/linux-generic/include/odp_time_internal.h
+++ b/platform/linux-generic/include/odp_time_internal.h
@@ -24,4 +24,6 @@ static inline uint64_t core_tick(void)
 #endif
 }
 
+uint64_t core_tick_diff_ns(uint64_t before, uint64_t after);
+
 #endif
diff --git a/platform/linux-generic/include/odp_timer_internal.h 
b/platform/linux-generic/include/odp_timer_internal.h
index b1cd73f..51959c0 100644
--- a/platform/linux-generic/include/odp_timer_internal.h
+++ b/platform/linux-generic/include/odp_timer_internal.h
@@ -39,4 +39,6 @@ typedef struct odp_timeout_hdr_stride {
        uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_timeout_hdr_t))];
 } odp_timeout_hdr_stride;
 
+int timer_run(void);
+
 #endif
diff --git a/platform/linux-generic/odp_schedule.c 
b/platform/linux-generic/odp_schedule.c
index 81e79c9..197fd72 100644
--- a/platform/linux-generic/odp_schedule.c
+++ b/platform/linux-generic/odp_schedule.c
@@ -22,6 +22,8 @@
 #include <odp_align_internal.h>
 #include <odp_schedule_internal.h>
 #include <odp_schedule_ordered_internal.h>
+#include <odp_time_internal.h>
+#include <odp_timer_internal.h>
 #include <odp/api/sync.h>
 
 /* Number of priority levels  */
@@ -788,19 +790,31 @@ static int do_schedule(odp_queue_t *out_queue, 
odp_event_t out_ev[],
        return 0;
 }
 
+#ifdef ODP_POLLED_TIMERS
+static __thread uint64_t last_timer_run;
+#endif
 
 static int schedule_loop(odp_queue_t *out_queue, uint64_t wait,
                         odp_event_t out_ev[],
                         unsigned int max_num)
 {
-       odp_time_t next, wtime;
-       int first = 1;
-       int ret;
-
-       while (1) {
-               ret = do_schedule(out_queue, out_ev, max_num);
+       uint64_t start, now;
+       int nr_events;
+
+       start = core_tick();
+
+       for (;;) {
+#ifdef ODP_POLLED_TIMERS
+               now = core_tick();
+               if (ODP_CONFIG_TIMER_RUN_NSEC <=
+                   core_tick_diff_ns(last_timer_run, now)) {
+                       last_timer_run = now;
+                       (void)timer_run();
+               }
+#endif
+               nr_events = do_schedule(out_queue, out_ev, max_num);
 
-               if (ret)
+               if (nr_events)
                        break;
 
                if (wait == ODP_SCHED_WAIT)
@@ -809,18 +823,12 @@ static int schedule_loop(odp_queue_t *out_queue, uint64_t 
wait,
                if (wait == ODP_SCHED_NO_WAIT)
                        break;
 
-               if (first) {
-                       wtime = odp_time_local_from_ns(wait);
-                       next = odp_time_sum(odp_time_local(), wtime);
-                       first = 0;
-                       continue;
-               }
-
-               if (odp_time_cmp(next, odp_time_local()) < 0)
+               now = core_tick();
+               if (core_tick_diff_ns(start, now) >= wait)
                        break;
        }
 
-       return ret;
+       return nr_events;
 }
 
 static odp_event_t schedule(odp_queue_t *out_queue, uint64_t wait)
diff --git a/platform/linux-generic/odp_time.c 
b/platform/linux-generic/odp_time.c
index 81e0522..1f14cf0 100644
--- a/platform/linux-generic/odp_time.c
+++ b/platform/linux-generic/odp_time.c
@@ -6,13 +6,68 @@
 
 #include <odp_posix_extensions.h>
 
-#include <time.h>
+#include <odp/api/cpu.h>
 #include <odp/api/time.h>
 #include <odp/api/hints.h>
 #include <odp_debug_internal.h>
+#include <odp_time_internal.h>
+
+#include <math.h>
+#include <time.h>
+#include <unistd.h>
+
+/* Number of nanoseconds per second. */
+#define ODP_NSEC_PER_SEC  (ODP_TIME_SEC_IN_NS)
+
+static uint64_t core_tick_hz;
 
 static odp_time_t start_time;
 
+uint64_t core_tick_diff_ns(uint64_t before, uint64_t after)
+{
+       uint64_t diff = after - before;
+
+       //TODO: khz instead of hz, skip llround
+       return llround(((double)diff / (double)core_tick_hz) *
+                      ODP_NSEC_PER_SEC);
+}
+
+static bool core_tick_init(void)
+{
+#if defined(__aarch64__)
+       uint64_t hz;
+       __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(hz));
+       core_tick_hz = hz;
+       return true;
+#elif defined(__x86_64__)
+       uint32_t cpu_info[4];
+
+       /* Check for Invariant TSC. */
+       __asm__ volatile("cpuid" :
+                        "=a"(cpu_info[0]),
+                        "=b"(cpu_info[1]),
+                        "=c"(cpu_info[2]),
+                        "=d"(cpu_info[3]) :
+                        "a"(0x80000000));
+       if (cpu_info[0] >= 0x80000007) {
+               __asm__ volatile("cpuid" :
+                                "=a"(cpu_info[0]),
+                                "=b"(cpu_info[1]),
+                                "=c"(cpu_info[2]),
+                                "=d"(cpu_info[3]) :
+                                "a"(0x80000007));
+               if (!(cpu_info[3] & (1 << 8)))
+                       return false;
+       }
+       //TODO: VM cpuid
+
+       core_tick_hz = odp_cpu_hz();
+       //TODO: libcpufreq
+       return true;
+#endif
+       return false;
+}
+
 static inline
 uint64_t time_to_ns(odp_time_t time)
 {
@@ -204,6 +259,9 @@ int odp_time_init_global(void)
                start_time.tv_nsec = time.tv_nsec;
        }
 
+       if (!core_tick_init())
+               ODP_ABORT("core_tick_init failed\n");
+
        return ret;
 }
 
diff --git a/platform/linux-generic/odp_timer.c 
b/platform/linux-generic/odp_timer.c
index ad44ede..81750e4 100644
--- a/platform/linux-generic/odp_timer.c
+++ b/platform/linux-generic/odp_timer.c
@@ -166,6 +166,9 @@ static inline void set_next_free(odp_timer *tim, uint32_t 
nf)
 
 typedef struct odp_timer_pool_s {
 /* Put frequently accessed fields in the first cache line */
+#ifdef ODP_POLLED_TIMERS
+       uint64_t cur_ts;
+#endif
        odp_atomic_u64_t cur_tick;/* Current tick value */
        uint64_t min_rel_tck;
        uint64_t max_rel_tck;
@@ -179,11 +182,13 @@ typedef struct odp_timer_pool_s {
        odp_timer_pool_param_t param;
        char name[ODP_TIMER_POOL_NAME_LEN];
        odp_shm_t shm;
+#ifndef ODP_POLLED_TIMERS
        timer_t timerid;
        int notify_overrun;
        pthread_t timer_thread; /* pthread_t of timer thread */
        pid_t timer_thread_id; /* gettid() for timer thread */
        int timer_thread_exit; /* request to exit for timer thread */
+#endif
 } odp_timer_pool;
 
 #define MAX_TIMER_POOLS 255 /* Leave one for ODP_TIMER_INVALID */
@@ -219,9 +224,11 @@ static inline odp_timer_t tp_idx_to_handle(struct 
odp_timer_pool_s *tp,
        return _odp_cast_scalar(odp_timer_t, (tp->tp_idx << INDEX_BITS) | idx);
 }
 
+#ifndef ODP_POLLED_TIMERS
 /* Forward declarations */
 static void itimer_init(odp_timer_pool *tp);
 static void itimer_fini(odp_timer_pool *tp);
+#endif
 
 static odp_timer_pool_t odp_timer_pool_new(const char *_name,
                                           const odp_timer_pool_param_t *param)
@@ -245,6 +252,9 @@ static odp_timer_pool_t odp_timer_pool_new(const char 
*_name,
                ODP_ABORT("%s: timer pool shm-alloc(%zuKB) failed\n",
                          _name, (sz0 + sz1 + sz2) / 1024);
        odp_timer_pool *tp = (odp_timer_pool *)odp_shm_addr(shm);
+#ifdef ODP_POLLED_TIMERS
+       __atomic_store_n(&tp->cur_ts, core_tick(), __ATOMIC_RELAXED);
+#endif
        odp_atomic_init_u64(&tp->cur_tick, 0);
        snprintf(tp->name, sizeof(tp->name), "%s", _name);
        tp->shm = shm;
@@ -254,7 +264,9 @@ static odp_timer_pool_t odp_timer_pool_new(const char 
*_name,
        tp->num_alloc = 0;
        odp_atomic_init_u32(&tp->high_wm, 0);
        tp->first_free = 0;
+#ifndef ODP_POLLED_TIMERS
        tp->notify_overrun = 1;
+#endif
        tp->tick_buf = (void *)((char *)odp_shm_addr(shm) + sz0);
        tp->timers = (void *)((char *)odp_shm_addr(shm) + sz0 + sz1);
        /* Initialize all odp_timer entries */
@@ -273,11 +285,14 @@ static odp_timer_pool_t odp_timer_pool_new(const char 
*_name,
        tp->tp_idx = tp_idx;
        odp_spinlock_init(&tp->lock);
        timer_pool[tp_idx] = tp;
+#ifndef ODP_POLLED_TIMERS
        if (tp->param.clk_src == ODP_CLOCK_CPU)
                itimer_init(tp);
+#endif
        return tp;
 }
 
+#ifndef ODP_POLLED_TIMERS
 static void block_sigalarm(void)
 {
        sigset_t sigset;
@@ -297,17 +312,20 @@ static void stop_timer_thread(odp_timer_pool *tp)
        if (ret != 0)
                ODP_ABORT("unable to join thread, err %d\n", ret);
 }
+#endif
 
 static void odp_timer_pool_del(odp_timer_pool *tp)
 {
        odp_spinlock_lock(&tp->lock);
        timer_pool[tp->tp_idx] = NULL;
 
+#ifndef ODP_POLLED_TIMERS
        /* Stop timer triggering */
        if (tp->param.clk_src == ODP_CLOCK_CPU)
                itimer_fini(tp);
 
        stop_timer_thread(tp);
+#endif
 
        if (tp->num_alloc != 0) {
                /* It's a programming error to attempt to destroy a */
@@ -673,6 +691,44 @@ static unsigned odp_timer_pool_expire(odp_timer_pool_t 
tpid, uint64_t tick)
        return nexp;
 }
 
+#ifdef ODP_POLLED_TIMERS
+
+int timer_run(void)
+{
+       odp_timer_pool *tp;
+       size_t i;
+       int nexp = 0;
+
+       for (i = 0; i < MAX_TIMER_POOLS; i++) {
+               tp = timer_pool[i];
+
+               if (tp == NULL) break;
+
+               /*
+                * Check the last time this timer pool was expired. If one or
+                * more periods have passed, attempt to expire it.
+                */
+               uint64_t tp_ts =
+                       __atomic_load_n(&tp->cur_ts, __ATOMIC_RELAXED);
+               uint64_t now = core_tick();
+
+               if (core_tick_diff_ns(tp_ts, now) >= 
odp_timer_pool_resolution(tp)) {
+                       if (__atomic_compare_exchange_n(
+                                   &tp->cur_ts, &tp_ts, now, false,
+                                   __ATOMIC_RELEASE, __ATOMIC_RELAXED)) {
+                               uint64_t tp_tick;
+                               tp_tick = odp_atomic_fetch_inc_u64(
+                                       &tp->cur_tick);
+                               nexp += odp_timer_pool_expire(tp, tp_tick + 1);
+                       }
+               }
+       }
+
+       return nexp;
+}
+
+#else
+
 /******************************************************************************
  * POSIX timer support
  * Functions that use Linux/POSIX per-process timers and related facilities
@@ -773,8 +829,8 @@ static void itimer_init(odp_timer_pool *tp)
        nsec = res - sec * ODP_TIME_SEC_IN_NS;
 
        memset(&ispec, 0, sizeof(ispec));
-       ispec.it_interval.tv_sec  = (time_t)sec;
-       ispec.it_interval.tv_nsec = (long)nsec;
+       ispec.it_internal.tv_sec  = (time_t)sec;
+       ispec.it_internal.tv_nsec = (long)nsec;
        ispec.it_value.tv_sec     = (time_t)sec;
        ispec.it_value.tv_nsec    = (long)nsec;
 
@@ -790,6 +846,8 @@ static void itimer_fini(odp_timer_pool *tp)
                          strerror(errno));
 }
 
+#endif
+
 /******************************************************************************
  * Public API functions
  * Some parameter checks and error messages
@@ -1013,8 +1071,9 @@ int odp_timer_init_global(void)
 #endif
        odp_atomic_init_u32(&num_timer_pools, 0);
 
+#ifndef ODP_POLLED_TIMERS
        block_sigalarm();
-
+#endif
        return 0;
 }
 
-- 
2.7.4

Reply via email to