On Mon, Oct 03, 2016 at 09:08:55PM +0200, Richard Cochran wrote: > On Thu, Sep 15, 2016 at 01:23:20PM +0100, Kieran Tyrrell wrote: > > Signed-off-by: Kieran Tyrrell <kie...@sienda.com> > > Overall, this is looking much better. I will come back to this with > some comments, but I would like to run some tests first, hopefully > this week or next.
Once again, thanks for providing the proof-of-concept. The patch has some technical issues, and those would be fixable. However, a little while ago I ran some tests to see how well this works, and the result convinced me that the performance of the PHC timers doesn't justify the implementation effort. Here is what I did. I started with Linux 4.1.33-rt38 and backported your patch. In addition, I hacked the SDPs on the i210 to show up as gpios under /sys. Then I used a simple program (see below) running at RT priority to toggle a gpio every 10 milliseconds according to the PTP time. This gpio output was time stamped on an input to the PTP master. During the test, I ran hackbench to simulate a load on the host. I ran ptp4l on the i210 host and synchronized to an external master connected via a crossover cable. (Prior to running this test, I verified the host was synchronized to within 200 nanoseconds using the PPS output.) I used phc2sys to synchronize the Linux system time to the i210 PHC. I ran the test program in two ways. 1. using nanosleep(CLOCK_REALTIME) 2. using the PHC timer provided by your patch. The resulting graphic shows the offsets from the 100 PPS on the gpio over a ten minute test. http://linuxptp.sourceforge.net/phc-timer/phc-timer-vs-nanosleep.png These are the statistics. | | nanosleep | PHC Timer | |--------+---------------+---------------| | min | +8.933000e+03 | +2.080500e+04 | | max | +2.962100e+04 | +4.319700e+04 | | pk-pk | +2.068800e+04 | +2.239200e+04 | | mean | +1.517014e+04 | +2.845244e+04 | | stddev | +2.449972e+03 | +3.137972e+03 | Both results display jitter that is to be expected using a program running under RT Linux. What is really interesting is the average offset. There is an addition 13 microsecond delay when using the PHC timer on the i210, but why? Remember that reading the PCIe bus costs microseconds for each operation. The additional overhead of accessing the card over PCIe completely dwarfs any additional time accuracy. So the conclusion is, using PHC + phc2sys + nanosleep provides the best performance that we can offer under Linux for globally synchronized distributed applications. Thanks, Richard --- /* * This program tests synchronization to a global time source * in a distributed system. * * Portions taken from the sgd test program. * Copyright (C) 2015 linutronix GmbH * * Portions taken from linux/Documentation/ptp/testptp.c. * Copyright (C) 2010 OMICRON electronics GmbH * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License along * with this program; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ #define _GNU_SOURCE /*for CPU_SET*/ #include <errno.h> #include <fcntl.h> #include <pthread.h> #include <sched.h> #include <signal.h> #include <stdio.h> #include <stdlib.h> #include <string.h> #include <sys/ioctl.h> #include <sys/stat.h> #include <sys/types.h> #include <unistd.h> #define PERIOD_SEC 0 #define PERIOD_NSEC 10000000 static int gpio_fd = -1, running = 1, toggle; static clockid_t get_clockid(int fd) { #define CLOCKFD 3 #define FD_TO_CLOCKID(fd) ((~(clockid_t) (fd) << 3) | CLOCKFD) return FD_TO_CLOCKID(fd); } static void gpio_pulse(void) { char c; c = '1'; write(gpio_fd, &c, sizeof(c)); c = '0'; write(gpio_fd, &c, sizeof(c)); } static void gpio_toggle(void) { static int even; char c; if (even) { c = '1'; write(gpio_fd, &c, sizeof(c)); } else { c = '0'; write(gpio_fd, &c, sizeof(c)); } even = 1 - even; } static void handle_alarm(int s) { if (toggle) gpio_toggle(); else gpio_pulse(); } static int install_handler(int signum, void (*handler)(int)) { struct sigaction action; sigset_t mask; /* Unblock the signal. */ sigemptyset(&mask); sigaddset(&mask, signum); sigprocmask(SIG_UNBLOCK, &mask, NULL); /* Install the signal handler. */ action.sa_handler = handler; action.sa_flags = 0; sigemptyset(&action.sa_mask); sigaction(signum, &action, NULL); return 0; } static int run_nanosleep(clockid_t clkid) { struct timespec ts; int err; clock_gettime(clkid, &ts); ts.tv_sec += 2; ts.tv_nsec = 0; while (running) { err = clock_nanosleep(clkid, TIMER_ABSTIME, &ts, NULL); switch (err) { case 0: if (toggle) { gpio_toggle(); } else { gpio_pulse(); } ts.tv_sec += PERIOD_SEC; ts.tv_nsec += PERIOD_NSEC; while (ts.tv_nsec > 999999999) { ts.tv_sec += 1; ts.tv_nsec -= 1000000000; } break; case EINTR: continue; default: fprintf(stderr, "clock_nanosleep returned %d: %s", err, strerror(err)); } } return 0; } static int run_timer(clockid_t clkid) { static timer_t timerid; struct itimerspec timeout; struct sigevent sigevent; install_handler(SIGALRM, handle_alarm); /* Create a timer. */ sigevent.sigev_notify = SIGEV_SIGNAL; sigevent.sigev_signo = SIGALRM; if (timer_create(clkid, &sigevent, &timerid)) { perror("timer_create"); return -1; } /* Start the timer. */ memset(&timeout, 0, sizeof(timeout)); timeout.it_interval.tv_sec = PERIOD_SEC; timeout.it_interval.tv_nsec = PERIOD_NSEC; clock_gettime(clkid, &timeout.it_value); timeout.it_value.tv_sec += 2; timeout.it_value.tv_nsec = 0; if (timer_settime(timerid, TIMER_ABSTIME, &timeout, NULL)) { perror("timer_settime"); return -1; } while (running) { pause(); } timer_delete(timerid); return 0; } static int set_realtime(pthread_t thread, int priority, int cpu) { cpu_set_t cpuset; struct sched_param sp; int err, policy; int min = sched_get_priority_min(SCHED_FIFO); int max = sched_get_priority_max(SCHED_FIFO); fprintf(stderr, "min %d max %d\n", min, max); if (priority < 0) { return 0; } err = pthread_getschedparam(thread, &policy, &sp); if (err) { fprintf(stderr, "pthread_getschedparam: %s\n", strerror(err)); return -1; } sp.sched_priority = priority; err = pthread_setschedparam(thread, SCHED_FIFO, &sp); if (err) { fprintf(stderr, "pthread_setschedparam: %s\n", strerror(err)); return -1; } if (cpu < 0) { return 0; } CPU_ZERO(&cpuset); CPU_SET(cpu, &cpuset); err = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset); if (err) { fprintf(stderr, "pthread_setaffinity_np: %s\n", strerror(err)); return -1; } return 0; } static void usage(char *progname) { fprintf(stderr, "\n" "usage: %s [options]\n" "\n" " -c [num] run on CPU 'num'\n" " -d [file] open device 'file'\n" " omit to use CLOCK_REALTIME instead\n" " -h prints this message and exits\n" " -n use clock_nanosleep() instead of timer_settime()\n" " -o [file] write to gpio 'file'\n" " -p [num] run with RT priorty 'num'\n" " -t toggle instead of making a pulse\n" "\n", progname); } int main(int argc, char *argv[]) { int c, cpu = -1, fd, err, nanosleep = 0, priority = -1; char *device = NULL, *gpio = NULL, *progname; clockid_t clkid; /* Process the command line arguments. */ progname = strrchr(argv[0], '/'); progname = progname ? 1 + progname : argv[0]; while (EOF != (c = getopt(argc, argv, "c:d:hno:p:t"))) { switch (c) { case 'c': cpu = atoi(optarg); break; case 'd': device = optarg; break; case 'h': usage(progname); return 0; case 'n': nanosleep = 1; break; case 'o': gpio = optarg; break; case 'p': priority = atoi(optarg); break; case 't': toggle = 1; break; case '?': usage(progname); return -1; } } if (!gpio) { usage(progname); return -1; } if (set_realtime(pthread_self(), priority, cpu)) { return -1; } gpio_fd = open(gpio, O_RDWR); if (gpio_fd < 0) { fprintf(stderr, "cannot open %s: %m\n", gpio); return -1; } if (device) { fd = open(device, O_RDWR); if (fd < 0) { fprintf(stderr, "cannot open %s: %m\n", device); return -1; } clkid = get_clockid(fd); } else { clkid = CLOCK_REALTIME; } if (nanosleep) { err = run_nanosleep(clkid); } else { err = run_timer(clkid); } if (device) { close(fd); } close(gpio_fd); return err; } ------------------------------------------------------------------------------ Developer Access Program for Intel Xeon Phi Processors Access to Intel Xeon Phi processor-based developer platforms. With one year of Intel Parallel Studio XE. Training and support from Colfax. Order your platform today.http://sdm.link/xeonphi _______________________________________________ Linuxptp-devel mailing list Linuxptp-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/linuxptp-devel
