From: Bogdan Pricope <bogdan.pric...@linaro.org>

Use low level PAPI API to get performance counters.
Exemplify on some ODP APIs.

Signed-off-by: Bogdan Pricope <bogdan.pric...@linaro.org>
---
/** Email created from pull request 406 (bogdanPricope:cat_benchmark_pr)
 ** https://github.com/Linaro/odp/pull/406
 ** Patch: https://github.com/Linaro/odp/pull/406.patch
 ** Base sha: 079b524277414f57411d3223563353bbdbc1a536
 ** Merge commit sha: c25517810ea3389a28c0adb2fdde601eae684851
 **/
 example/instrum/Makefile.am      |  14 +++-
 example/instrum/init.c           |  54 +++++++++++++++
 example/instrum/init.h           |  19 ++++++
 example/instrum/instrum.c        |  19 +++++-
 example/instrum/instrum_common.h |  34 ++++++++++
 example/instrum/papi_cnt.c       | 143 +++++++++++++++++++++++++++++++++++++++
 example/instrum/papi_cnt.h       |  27 ++++++++
 example/instrum/pktio_direct.c   |  68 +++++++++++++++++++
 example/instrum/pktio_direct.h   |  20 ++++++
 example/instrum/sample.h         |  28 ++++++++
 example/instrum/sched.c          |  43 ++++++++++++
 example/instrum/sched.h          |  19 ++++++
 example/instrum/store.c          | 133 ++++++++++++++++++++++++++++++++++++
 example/instrum/store.h          |  36 ++++++++++
 14 files changed, 655 insertions(+), 2 deletions(-)
 create mode 100644 example/instrum/init.c
 create mode 100644 example/instrum/init.h
 create mode 100644 example/instrum/instrum_common.h
 create mode 100644 example/instrum/papi_cnt.c
 create mode 100644 example/instrum/papi_cnt.h
 create mode 100644 example/instrum/pktio_direct.c
 create mode 100644 example/instrum/pktio_direct.h
 create mode 100644 example/instrum/sample.h
 create mode 100644 example/instrum/sched.c
 create mode 100644 example/instrum/sched.h
 create mode 100644 example/instrum/store.c
 create mode 100644 example/instrum/store.h

diff --git a/example/instrum/Makefile.am b/example/instrum/Makefile.am
index bf2a19c0d..42c8161bf 100644
--- a/example/instrum/Makefile.am
+++ b/example/instrum/Makefile.am
@@ -17,4 +17,16 @@ AM_LDFLAGS = -L$(PAPI_PATH)/lib -lpapi
 lib_LTLIBRARIES = $(LIB)/libinstrum.la
 
 __LIB__libinstrum_la_SOURCES = \
-               instrum.c
+               instrum.c \
+               store.c \
+               papi_cnt.c \
+               init.c \
+               pktio_direct.c\
+               sched.c \
+               instrum_common.h \
+               sample.h \
+               store.h \
+               papi_cnt.h \
+               init.h \
+               pktio_direct.h \
+               sched.h
diff --git a/example/instrum/init.c b/example/instrum/init.c
new file mode 100644
index 000000000..8691ff978
--- /dev/null
+++ b/example/instrum/init.c
@@ -0,0 +1,54 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <init.h>
+#include <store.h>
+
+static int (*instr_odp_init_local)(odp_instance_t instance,
+                                  odp_thread_type_t thr_type);
+
+static int (*instr_odp_term_local)(void);
+
+int instr_odpinit_init(void)
+{
+       INSTR_FUNCTION(odp_init_local);
+
+       if (!instr_odp_init_local) {
+               printf("odp_init_local: Not Found\n");
+               return -1;
+       }
+
+       INSTR_FUNCTION(odp_term_local);
+
+       if (!instr_odp_term_local) {
+               printf("odp_term_local: Not Found\n");
+               return -1;
+       }
+
+       return 0;
+}
+
+int odp_init_local(odp_instance_t instance, odp_thread_type_t thr_type)
+{
+       int ret;
+
+       ret = (*instr_odp_init_local)(instance, thr_type);
+
+       instr_store_init_local();
+
+       return ret;
+}
+
+int odp_term_local(void)
+{
+       instr_store_term_local();
+
+       return (*instr_odp_term_local)();
+}
diff --git a/example/instrum/init.h b/example/instrum/init.h
new file mode 100644
index 000000000..b92e9d4a3
--- /dev/null
+++ b/example/instrum/init.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_INIT_H__
+#define __INSTRUM_INIT_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odpinit_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /*__INSTRUM_INIT_H__*/
diff --git a/example/instrum/instrum.c b/example/instrum/instrum.c
index 88580c58f..605675154 100644
--- a/example/instrum/instrum.c
+++ b/example/instrum/instrum.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017, Linaro Limited
+/* Copyright (c) 2018, Linaro Limited
  * All rights reserved.
  *
  * SPDX-License-Identifier:     BSD-3-Clause
@@ -6,13 +6,30 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <store.h>
+#include <init.h>
+#include <sched.h>
+#include <pktio_direct.h>
 
 static __attribute__((constructor)) void setup_wrappers(void)
 {
        printf("Setup Wrappers\n");
+
+       if (instr_store_init())
+               return;
+
+       if (instr_odpinit_init())
+               return;
+
+       if (instr_odpsched_init())
+               return;
+
+       if (instr_odppktio_direct_init())
+               return;
 }
 
 static __attribute__((destructor)) void teardown_wrappers(void)
 {
        printf("Teardown Wrappers\n");
+       instr_store_term();
 }
diff --git a/example/instrum/instrum_common.h b/example/instrum/instrum_common.h
new file mode 100644
index 000000000..594423e1f
--- /dev/null
+++ b/example/instrum/instrum_common.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_COMMON_H__
+#define __INSTRUM_COMMON_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef RTLD_NEXT
+/*#define __GNU_SOURCE*/
+#define __USE_GNU
+#endif
+
+#include <dlfcn.h>
+#include <errno.h>
+
+#define INSTR_FUNCTION(func) do {                      \
+               instr_##func = dlsym(RTLD_NEXT, #func); \
+               if (dlerror()) {                        \
+                       errno = EACCES;                 \
+                       instr_##func = NULL;    \
+               }                                       \
+       } while (0)
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __INSTRUM_COMMON_H__ */
diff --git a/example/instrum/papi_cnt.c b/example/instrum/papi_cnt.c
new file mode 100644
index 000000000..c38c856a0
--- /dev/null
+++ b/example/instrum/papi_cnt.c
@@ -0,0 +1,143 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <papi.h>
+#include <papi_cnt.h>
+
+static int papi_event_tab[SAMPLE_COUNTER_TAB_SIZE] = {PAPI_BR_CN, PAPI_L2_DCM};
+
+static __thread int event_set = PAPI_NULL;
+
+int papi_init(void)
+{
+       int retval, i;
+
+       retval = PAPI_library_init(PAPI_VER_CURRENT);
+       if (retval != PAPI_VER_CURRENT) {
+               printf("PAPI Library initialization error!\n");
+               return -1;
+       }
+
+       retval = PAPI_thread_init((unsigned long(*)(void))(pthread_self));
+       if (retval != PAPI_OK) {
+               printf("PAPI_thread_init error!\n");
+               goto err_shutdown;
+       }
+
+       if (PAPI_set_granularity(PAPI_GRN_THR) != PAPI_OK) {
+               printf("PAPI_set_granularity error!\n");
+               goto err_shutdown;
+       }
+
+       for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++) {
+               retval = PAPI_query_event(papi_event_tab[i]);
+               if (retval != PAPI_OK) {
+                       printf("PAPI_query_event %d - error\n", i);
+                       goto err_shutdown;
+               }
+       }
+
+       return 0;
+
+err_shutdown:
+       PAPI_shutdown();
+
+       return -1;
+}
+
+void papi_term(void)
+{
+       PAPI_shutdown();
+}
+
+int papi_init_local(void)
+{
+       int retval;
+
+       retval = PAPI_register_thread();
+       if (retval != PAPI_OK) {
+               printf("PAPI_register_thread failed - %d\n", retval);
+               return -1;
+       }
+
+       /* Create LL event set */
+       event_set = PAPI_NULL;
+       retval = PAPI_create_eventset(&event_set);
+       if (retval != PAPI_OK) {
+               printf("PAPI_create_eventset error: %d\n", retval);
+               return -1;
+       }
+
+       retval = PAPI_add_events(event_set, papi_event_tab,
+                                SAMPLE_COUNTER_TAB_SIZE);
+       if (retval != PAPI_OK) {
+               printf("PAPI_add_events error: %d\n", retval);
+               goto err_clean_evset;
+       }
+
+       retval = PAPI_start(event_set);
+       if (retval != PAPI_OK) {
+               printf("PAPI_start error: %d\n", retval);
+               goto err_clean_evset;
+       }
+
+       return 0;
+
+err_clean_evset:
+       PAPI_cleanup_eventset(event_set);
+       PAPI_destroy_eventset(&event_set);
+
+       return -1;
+}
+
+int papi_term_local(void)
+{
+       long long last_counters[SAMPLE_COUNTER_TAB_SIZE];
+
+       if (PAPI_stop(event_set, last_counters) == PAPI_OK) {
+               int i;
+
+               for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++)
+                       printf("Counter[%d] = %lld\n", i, last_counters[i]);
+       }
+
+       PAPI_cleanup_eventset(event_set);
+       PAPI_destroy_eventset(&event_set);
+
+       return 0;
+}
+
+int papi_sample_start(profiling_sample_t *spl)
+{
+       spl->timestamp_ns = PAPI_get_real_nsec();
+       if (PAPI_read_ts(event_set, spl->counters, &spl->diff_cyc) != PAPI_OK) {
+               fprintf(stderr, "PAPI_read_counters - FAILED\n");
+               return -1;
+       }
+
+       return 0;
+}
+
+int papi_sample_end(profiling_sample_t *spl)
+{
+       long long end_counters[SAMPLE_COUNTER_TAB_SIZE], end_cyc;
+       int i;
+
+       if (PAPI_read_ts(event_set, end_counters, &end_cyc) != PAPI_OK) {
+               fprintf(stderr, "PAPI_read_counters - FAILED\n");
+               return -1;
+       }
+
+       for (i = 0; i < SAMPLE_COUNTER_TAB_SIZE; i++)
+               spl->counters[i] = end_counters[i] - spl->counters[i];
+
+       spl->diff_cyc = end_cyc - spl->diff_cyc;
+
+       return 0;
+}
diff --git a/example/instrum/papi_cnt.h b/example/instrum/papi_cnt.h
new file mode 100644
index 000000000..a4546c954
--- /dev/null
+++ b/example/instrum/papi_cnt.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_PAPI_COUNTERS_H__
+#define __INSTRUM_PAPI_COUNTERS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <sample.h>
+
+int papi_init(void);
+void papi_term(void);
+int papi_init_local(void);
+int papi_term_local(void);
+
+int papi_sample_start(profiling_sample_t *spl);
+int papi_sample_end(profiling_sample_t *spl);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_PAPI_COUNTERS_H__ */
diff --git a/example/instrum/pktio_direct.c b/example/instrum/pktio_direct.c
new file mode 100644
index 000000000..1cef986d5
--- /dev/null
+++ b/example/instrum/pktio_direct.c
@@ -0,0 +1,68 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <pktio_direct.h>
+#include <store.h>
+
+static int (*instr_odp_pktout_send)(odp_pktout_queue_t queue,
+                                   const odp_packet_t packets[],
+                                   int num);
+
+static int (*instr_odp_pktin_recv_tmo)(odp_pktin_queue_t queue,
+                                      odp_packet_t packets[],
+                                      int num, uint64_t wait);
+
+int instr_odppktio_direct_init(void)
+{
+       INSTR_FUNCTION(odp_pktout_send);
+
+       if (!instr_odp_pktout_send) {
+               printf("odp_pktout_send: Not Found\n");
+               return -1;
+       }
+
+       INSTR_FUNCTION(odp_pktin_recv_tmo);
+
+       if (!instr_odp_pktin_recv_tmo) {
+               printf("odp_pktin_recv_tmo: Not Found\n");
+               return -1;
+       }
+
+       return 0;
+}
+
+int odp_pktout_send(odp_pktout_queue_t queue, const odp_packet_t packets[],
+                   int num)
+{
+       int ret;
+
+       STORE_SAMPLE_INIT;
+
+       STORE_SAMPLE_START;
+       ret = (*instr_odp_pktout_send)(queue, packets, num);
+       STORE_SAMPLE_END;
+
+       return ret;
+}
+
+int odp_pktin_recv_tmo(odp_pktin_queue_t queue, odp_packet_t packets[],
+                      int num, uint64_t wait)
+{
+       int ret;
+
+       STORE_SAMPLE_INIT;
+
+       STORE_SAMPLE_START;
+       ret = (*instr_odp_pktin_recv_tmo)(queue, packets, num, wait);
+       STORE_SAMPLE_END;
+
+       return ret;
+}
+
diff --git a/example/instrum/pktio_direct.h b/example/instrum/pktio_direct.h
new file mode 100644
index 000000000..b028c3a4f
--- /dev/null
+++ b/example/instrum/pktio_direct.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_PKTIO_DIRECT_H__
+#define __INSTRUM_PKTIO_DIRECT_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odppktio_direct_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_PKTIO_DIRECT_H__ */
+
diff --git a/example/instrum/sample.h b/example/instrum/sample.h
new file mode 100644
index 000000000..090308348
--- /dev/null
+++ b/example/instrum/sample.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_SAMPLE_H__
+#define __INSTRUM_SAMPLE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SAMPLE_NAME_SIZE_MAX 20
+#define SAMPLE_COUNTER_TAB_SIZE 2
+
+typedef struct {
+       char name[SAMPLE_NAME_SIZE_MAX];
+       long long timestamp_ns;
+       long long diff_cyc;
+
+       long long counters[SAMPLE_COUNTER_TAB_SIZE];
+} profiling_sample_t;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_SAMPLE_H__ */
diff --git a/example/instrum/sched.c b/example/instrum/sched.c
new file mode 100644
index 000000000..3e4a03bf1
--- /dev/null
+++ b/example/instrum/sched.c
@@ -0,0 +1,43 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <instrum_common.h>
+#include <sched.h>
+#include <store.h>
+
+static int (*instr_odp_schedule_multi)(odp_queue_t *from,
+                                      uint64_t wait,
+                                      odp_event_t events[],
+                                      int num);
+
+int instr_odpsched_init(void)
+{
+       INSTR_FUNCTION(odp_schedule_multi);
+
+       if (!instr_odp_schedule_multi) {
+               printf("odp_schedule_multi: Not Found\n");
+               return -1;
+       }
+
+       return 0;
+}
+
+int odp_schedule_multi(odp_queue_t *from, uint64_t wait, odp_event_t events[],
+                      int num)
+{
+       int ret;
+
+       STORE_SAMPLE_INIT;
+
+       STORE_SAMPLE_START;
+       ret = (*instr_odp_schedule_multi)(from, wait, events, num);
+       STORE_SAMPLE_END;
+
+       return ret;
+}
diff --git a/example/instrum/sched.h b/example/instrum/sched.h
new file mode 100644
index 000000000..2a092b0ed
--- /dev/null
+++ b/example/instrum/sched.h
@@ -0,0 +1,19 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_SCHED_H__
+#define __INSTRUM_SCHED_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int instr_odpsched_init(void);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_SCHED_H__ */
diff --git a/example/instrum/store.c b/example/instrum/store.c
new file mode 100644
index 000000000..77137aad8
--- /dev/null
+++ b/example/instrum/store.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <odp_api.h>
+#include <store.h>
+#include <sample.h>
+#include <papi_cnt.h>
+
+#define SAMPLE_TAB_SIZE 50000
+
+static __thread profiling_sample_t profile_sample_tab[SAMPLE_TAB_SIZE];
+static __thread uint64_t profile_sample_idx;
+static __thread uint64_t profile_sample_ovf;
+
+#define STORE_DIR_ENV "ODP_INSTRUM_STORE_DIR"
+#define STORE_DIR_NAME_DFLT "/tmp"
+#define STORE_DIR_NAME_SIZE_MAX 250
+#define STORE_FILE_NAME_SIZE_MAX 250
+
+static char store_dir[STORE_DIR_NAME_SIZE_MAX];
+
+static void store_dump(int last)
+{
+       FILE *f = NULL;
+       char file_name[STORE_DIR_NAME_SIZE_MAX + STORE_FILE_NAME_SIZE_MAX];
+       char smpl[250], smpl_tmp[250];
+       int i, j, dump_size = SAMPLE_TAB_SIZE;
+
+       if (last)
+               dump_size = profile_sample_idx;
+
+       sprintf(file_name, "%s/profile_%d_%ju.csv",
+               store_dir, odp_thread_id(),
+               profile_sample_ovf);
+
+       f = fopen(file_name, "w");
+       if (f == NULL) {
+               printf("Failed to create profiling file %s\n", file_name);
+               return;
+       }
+
+       for (i = 0; i < dump_size; i++) {
+               sprintf(smpl, "%lld,%lld,%s",
+                       profile_sample_tab[i].timestamp_ns,
+                       profile_sample_tab[i].diff_cyc,
+                       profile_sample_tab[i].name);
+               for (j = 0; j < SAMPLE_COUNTER_TAB_SIZE; j++) {
+                       sprintf(smpl_tmp, ",%lld",
+                               profile_sample_tab[i].counters[j]);
+                       strcat(smpl, smpl_tmp);
+               }
+               fprintf(f, "%s\n", smpl);
+       }
+
+       fclose(f);
+}
+
+int instr_store_init(void)
+{
+       const char *store_dir_env = NULL;
+
+       store_dir_env = getenv(STORE_DIR_ENV);
+       if (!store_dir_env)
+               store_dir_env = STORE_DIR_NAME_DFLT;
+
+       strncpy(store_dir, store_dir_env, STORE_DIR_NAME_SIZE_MAX);
+       store_dir[STORE_DIR_NAME_SIZE_MAX - 1] = '\0';
+
+       if (papi_init())
+               return -1;
+
+       return 0;
+}
+
+void instr_store_term(void)
+{
+       papi_term();
+}
+
+int instr_store_init_local(void)
+{
+       return papi_init_local();
+}
+
+int instr_store_term_local(void)
+{
+       int ret = papi_term_local();
+
+       store_dump(1);
+
+       return ret;
+}
+
+instr_profiling_sample_t store_sample_start(const char *func)
+{
+       profiling_sample_t *spl = NULL;
+
+       if (profile_sample_idx == SAMPLE_TAB_SIZE)
+               return NULL;
+
+       spl = &profile_sample_tab[profile_sample_idx];
+
+       strncpy(spl->name, func, SAMPLE_NAME_SIZE_MAX);
+       spl->name[SAMPLE_NAME_SIZE_MAX - 1] = '\0';
+
+       if (papi_sample_start(spl))
+               return NULL;
+
+       profile_sample_idx++;
+       return spl;
+}
+
+void store_sample_end(instr_profiling_sample_t _spl)
+{
+       profiling_sample_t *spl = _spl;
+
+       if (!spl) /* failed sample - on start */
+               return;
+
+       if (papi_sample_end(spl))
+               spl->name[0] = 0; /* failed sample - on end*/
+
+       if (profile_sample_idx == SAMPLE_TAB_SIZE) {
+               store_dump(0);
+               profile_sample_idx = 0;
+               profile_sample_ovf++;
+       }
+}
diff --git a/example/instrum/store.h b/example/instrum/store.h
new file mode 100644
index 000000000..1d671fbfd
--- /dev/null
+++ b/example/instrum/store.h
@@ -0,0 +1,36 @@
+/* Copyright (c) 2018, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#ifndef __INSTRUM_STORE_H__
+#define __INSTRUM_STORE_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void *instr_profiling_sample_t;
+
+#define STORE_SAMPLE_INIT \
+       instr_profiling_sample_t _spl
+
+#define STORE_SAMPLE_START \
+       (_spl = store_sample_start(__func__))
+
+#define STORE_SAMPLE_END \
+       store_sample_end(_spl)
+
+int instr_store_init(void);
+void instr_store_term(void);
+int instr_store_init_local(void);
+int instr_store_term_local(void);
+
+instr_profiling_sample_t store_sample_start(const char *);
+void store_sample_end(instr_profiling_sample_t);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* __INSTRUM_STORE_H__ */

Reply via email to