Hello community, here is the log from the commit of package fio for openSUSE:Factory checked in at 2018-02-20 17:55:12 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/fio (Old) and /work/SRC/openSUSE:Factory/.fio.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "fio" Tue Feb 20 17:55:12 2018 rev:43 rq:578033 version:3.4 Changes: -------- --- /work/SRC/openSUSE:Factory/fio/fio.changes 2018-01-10 23:34:48.557033304 +0100 +++ /work/SRC/openSUSE:Factory/.fio.new/fio.changes 2018-02-20 17:55:25.809697710 +0100 @@ -1,0 +2,8 @@ +Sun Feb 18 09:07:00 UTC 2018 - [email protected] + +- Update to version 3.4: + * For detailed list of changes see: + http://brick.kernel.dk/snaps/fio-3.4.shortlog.txt +- simplify build and install + +------------------------------------------------------------------- Old: ---- fio-3.3.tar.bz2 New: ---- fio-3.4.tar.bz2 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ fio.spec ++++++ --- /var/tmp/diff_new_pack.1uh77d/_old 2018-02-20 17:55:26.685666169 +0100 +++ /var/tmp/diff_new_pack.1uh77d/_new 2018-02-20 17:55:26.689666025 +0100 @@ -18,12 +18,12 @@ Name: fio -Version: 3.3 +Version: 3.4 Release: 0 Summary: Flexible I/O tester License: GPL-2.0 Group: System/Benchmark -URL: http://git.kernel.dk/?p=fio.git;a=summary +Url: http://git.kernel.dk/?p=fio.git;a=summary Source: http://brick.kernel.dk/snaps/fio-%{version}.tar.bz2 BuildRequires: gtk2-devel BuildRequires: libaio-devel @@ -73,21 +73,12 @@ %{?_smp_mflags} \ V=1 \ OPTFLAGS="%{optflags}" \ - CC="cc" \ - prefix="%{_prefix}" \ - libdir="%{_libdir}/fio" \ - mandir="%{_mandir}" + CC="cc" %install -make \ - %{?_smp_mflags} \ - V=1 \ - DESTDIR=%{buildroot} \ +%make_install \ prefix="%{_prefix}" \ - bindir="%{_bindir}" \ - libdir="%{_libdir}/fio" \ - mandir="%{_mandir}" \ - install + mandir="%{_mandir}" %check make %{?_smp_mflags} test ++++++ fio-3.3.tar.bz2 -> fio-3.4.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/HOWTO new/fio-3.4/HOWTO --- old/fio-3.3/HOWTO 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/HOWTO 2018-02-12 18:55:07.000000000 +0100 @@ -1093,8 +1093,9 @@ .. option:: fadvise_hint=str - Use :manpage:`posix_fadvise(2)` to advise the kernel on what I/O patterns - are likely to be issued. Accepted values are: + Use :manpage:`posix_fadvise(2)` or :manpage:`posix_fadvise(2)` to + advise the kernel on what I/O patterns are likely to be issued. + Accepted values are: **0** Backwards-compatible hint for "no hint". diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/README new/fio-3.4/README --- old/fio-3.3/README 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/README 2018-02-12 18:55:07.000000000 +0100 @@ -120,7 +120,9 @@ Windows: Rebecca Cran <[email protected]> has fio packages for Windows at - http://www.bluestop.org/fio/ . + https://www.bluestop.org/fio/ . The latest builds for Windows can also + be grabbed from https://ci.appveyor.com/project/axboe/fio by clicking + the latest x86 or x64 build, then selecting the ARTIFACTS tab. BSDs: Packages for BSDs may be available from their binary package repositories. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/client.c new/fio-3.4/client.c --- old/fio-3.3/client.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/client.c 2018-02-12 18:55:07.000000000 +0100 @@ -1024,6 +1024,7 @@ client_ts.thread_number = p->ts.thread_number; client_ts.groupid = p->ts.groupid; client_ts.unified_rw_rep = p->ts.unified_rw_rep; + client_ts.sig_figs = p->ts.sig_figs; if (++sum_stat_nr == sum_stat_clients) { strcpy(client_ts.name, "All clients"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/debug.c new/fio-3.4/debug.c --- old/fio-3.3/debug.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/debug.c 2018-02-12 18:55:07.000000000 +0100 @@ -7,20 +7,11 @@ void __dprint(int type, const char *str, ...) { va_list args; - pid_t pid; assert(type < FD_DEBUG_MAX); - pid = getpid(); - if (fio_debug_jobp && *fio_debug_jobp != -1U - && pid != *fio_debug_jobp) - return; - - log_info("%-8s ", debug_levels[type].name); - log_info("%-5u ", (int) pid); - va_start(args, str); - log_valist(str, args); + log_prevalist(type, str, args); va_end(args); } #endif diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/debug.h new/fio-3.4/debug.h --- old/fio-3.3/debug.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/debug.h 2018-02-12 18:55:07.000000000 +0100 @@ -2,6 +2,7 @@ #define FIO_DEBUG_H #include <assert.h> +#include "lib/types.h" #include "log.h" enum { @@ -26,7 +27,23 @@ FD_DEBUG_MAX, }; -extern unsigned int fio_debug_jobno, *fio_debug_jobp; +extern unsigned int fio_debug_jobno, *fio_debug_jobp, *fio_warned; + +static inline bool fio_did_warn(unsigned int mask) +{ + if (*fio_warned & mask) + return true; + + *fio_warned |= mask; + return false; +} + +enum { + FIO_WARN_ROOT_FLUSH = 1, + FIO_WARN_VERIFY_BUF = 2, + FIO_WARN_ZONED_BUG = 4, + FIO_WARN_IOLOG_DROP = 8, +}; #ifdef FIO_INC_DEBUG struct debug_level { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/engines/dev-dax.c new/fio-3.4/engines/dev-dax.c --- old/fio-3.3/engines/dev-dax.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/engines/dev-dax.c 2018-02-12 18:55:07.000000000 +0100 @@ -157,7 +157,7 @@ * It fits within existing mapping, use it */ if (io_u->offset >= fdd->devdax_off && - io_u->offset + io_u->buflen < fdd->devdax_off + fdd->devdax_sz) + io_u->offset + io_u->buflen <= fdd->devdax_off + fdd->devdax_sz) goto done; /* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/engines/libpmem.c new/fio-3.4/engines/libpmem.c --- old/fio-3.3/engines/libpmem.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/engines/libpmem.c 2018-02-12 18:55:07.000000000 +0100 @@ -430,7 +430,7 @@ io_u->buflen, fdd->libpmem_sz); if (io_u->offset >= fdd->libpmem_off && - (io_u->offset + io_u->buflen < + (io_u->offset + io_u->buflen <= fdd->libpmem_off + fdd->libpmem_sz)) goto done; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/engines/mmap.c new/fio-3.4/engines/mmap.c --- old/fio-3.3/engines/mmap.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/engines/mmap.c 2018-02-12 18:55:07.000000000 +0100 @@ -27,15 +27,39 @@ off_t mmap_off; }; +static bool fio_madvise_file(struct thread_data *td, struct fio_file *f, + size_t length) + +{ + struct fio_mmap_data *fmd = FILE_ENG_DATA(f); + + if (!td->o.fadvise_hint) + return true; + + if (!td_random(td)) { + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) { + td_verror(td, errno, "madvise"); + return false; + } + } else { + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) { + td_verror(td, errno, "madvise"); + return false; + } + } + + return true; +} + static int fio_mmap_file(struct thread_data *td, struct fio_file *f, size_t length, off_t off) { struct fio_mmap_data *fmd = FILE_ENG_DATA(f); int flags = 0; - if (td_rw(td)) + if (td_rw(td) && !td->o.verify_only) flags = PROT_READ | PROT_WRITE; - else if (td_write(td)) { + else if (td_write(td) && !td->o.verify_only) { flags = PROT_WRITE; if (td->o.verify != VERIFY_NONE) @@ -50,17 +74,9 @@ goto err; } - if (!td_random(td)) { - if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_SEQUENTIAL) < 0) { - td_verror(td, errno, "madvise"); - goto err; - } - } else { - if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_RANDOM) < 0) { - td_verror(td, errno, "madvise"); - goto err; - } - } + if (!fio_madvise_file(td, f, length)) + goto err; + if (posix_madvise(fmd->mmap_ptr, length, POSIX_MADV_DONTNEED) < 0) { td_verror(td, errno, "madvise"); goto err; @@ -137,7 +153,7 @@ * It fits within existing mapping, use it */ if (io_u->offset >= fmd->mmap_off && - io_u->offset + io_u->buflen < fmd->mmap_off + fmd->mmap_sz) + io_u->offset + io_u->buflen <= fmd->mmap_off + fmd->mmap_sz) goto done; /* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/engines/null.c new/fio-3.4/engines/null.c --- old/fio-3.3/engines/null.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/engines/null.c 2018-02-12 18:55:07.000000000 +0100 @@ -87,9 +87,9 @@ } } -static int null_init(struct thread_data *td, struct null_data **nd_ptr) +static struct null_data *null_init(struct thread_data *td) { - struct null_data *nd = (struct null_data *) malloc(sizeof(**nd_ptr)); + struct null_data *nd = (struct null_data *) malloc(sizeof(*nd)); memset(nd, 0, sizeof(*nd)); @@ -99,47 +99,48 @@ } else td->io_ops->flags |= FIO_SYNCIO; - *nd_ptr = nd; - return 0; + return nd; } #ifndef __cplusplus static struct io_u *fio_null_event(struct thread_data *td, int event) { - return null_event((struct null_data *)td->io_ops_data, event); + return null_event(td->io_ops_data, event); } static int fio_null_getevents(struct thread_data *td, unsigned int min_events, unsigned int max, const struct timespec *t) { - struct null_data *nd = (struct null_data *)td->io_ops_data; + struct null_data *nd = td->io_ops_data; return null_getevents(nd, min_events, max, t); } static int fio_null_commit(struct thread_data *td) { - return null_commit(td, (struct null_data *)td->io_ops_data); + return null_commit(td, td->io_ops_data); } static int fio_null_queue(struct thread_data *td, struct io_u *io_u) { - return null_queue(td, (struct null_data *)td->io_ops_data, io_u); + return null_queue(td, td->io_ops_data, io_u); } static int fio_null_open(struct thread_data *td, struct fio_file *f) { - return null_open((struct null_data *)td->io_ops_data, f); + return null_open(td->io_ops_data, f); } static void fio_null_cleanup(struct thread_data *td) { - null_cleanup((struct null_data *)td->io_ops_data); + null_cleanup(td->io_ops_data); } static int fio_null_init(struct thread_data *td) { - return null_init(td, (struct null_data **)&td->io_ops_data); + td->io_ops_data = null_init(td); + assert(td->io_ops_data); + return 0; } static struct ioengine_ops ioengine = { @@ -172,7 +173,8 @@ struct NullData { NullData(struct thread_data *td) { - null_init(td, &impl_); + impl_ = null_init(td); + assert(impl_); } ~NullData() @@ -211,6 +213,7 @@ return null_open(impl_, f); } +private: struct null_data *impl_; }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/eta.c new/fio-3.4/eta.c --- old/fio-3.3/eta.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/eta.c 2018-02-12 18:55:07.000000000 +0100 @@ -9,7 +9,7 @@ #include "lib/pow2.h" static char __run_str[REAL_MAX_JOBS + 1]; -static char run_str[__THREAD_RUNSTR_SZ(REAL_MAX_JOBS)]; +static char run_str[__THREAD_RUNSTR_SZ(REAL_MAX_JOBS) + 1]; static void update_condensed_str(char *rstr, char *run_str_condensed) { @@ -520,7 +520,7 @@ static int eta_new_line_init, eta_new_line_pending; static int linelen_last; static int eta_good; - char output[REAL_MAX_JOBS + 512], *p = output; + char output[__THREAD_RUNSTR_SZ(REAL_MAX_JOBS) + 512], *p = output; char eta_str[128]; double perc = 0.0; @@ -531,6 +531,7 @@ if (eta_new_line_pending) { eta_new_line_pending = 0; + linelen_last = 0; p += sprintf(p, "\n"); } @@ -564,6 +565,7 @@ size_t left; int l; int ddir; + int linelen; if ((!je->eta_sec && !eta_good) || je->nr_ramp == je->nr_running || je->eta_sec == -1) @@ -601,10 +603,14 @@ rate_str[DDIR_READ], rate_str[DDIR_WRITE], iops_str[DDIR_READ], iops_str[DDIR_WRITE], eta_str); + /* If truncation occurred adjust l so p is on the null */ + if (l >= left) + l = left - 1; p += l; - if (l >= 0 && l < linelen_last) - p += sprintf(p, "%*s", linelen_last - l, ""); - linelen_last = l; + linelen = p - output; + if (l >= 0 && linelen < linelen_last) + p += sprintf(p, "%*s", linelen_last - linelen, ""); + linelen_last = linelen; for (ddir = 0; ddir < DDIR_RWDIR_CNT; ddir++) { free(rate_str[ddir]); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/examples/fio-seq-read.job new/fio-3.4/examples/fio-seq-read.job --- old/fio-3.3/examples/fio-seq-read.job 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/examples/fio-seq-read.job 2018-02-12 18:55:07.000000000 +0100 @@ -3,7 +3,7 @@ filename=fio-seq-reads rw=read bs=256K -direct=0 +direct=1 numjobs=1 time_based=1 runtime=900 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/filesetup.c new/fio-3.4/filesetup.c --- old/fio-3.3/filesetup.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/filesetup.c 2018-02-12 18:55:07.000000000 +0100 @@ -20,8 +20,6 @@ #include <linux/falloc.h> #endif -static int root_warn; - static FLIST_HEAD(filename_list); /* @@ -490,6 +488,10 @@ ret = td->io_ops->invalidate(td, f); if (ret < 0) errval = -ret; + } else if (td_ioengine_flagged(td, FIO_DISKLESSIO)) { + dprint(FD_IO, "invalidate not supported by ioengine %s\n", + td->io_ops->name); + ret = 0; } else if (f->filetype == FIO_TYPE_FILE) { dprint(FD_IO, "declare unneeded cache %s: %llu/%llu\n", f->file_name, off, len); @@ -512,10 +514,9 @@ ret = blockdev_invalidate_cache(f); } if (ret < 0 && errno == EACCES && geteuid()) { - if (!root_warn) { + if (!fio_did_warn(FIO_WARN_ROOT_FLUSH)) { log_err("fio: only root may flush block " "devices. Cache flush bypassed!\n"); - root_warn = 1; } ret = 0; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/fio.1 new/fio-3.4/fio.1 --- old/fio-3.3/fio.1 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/fio.1 2018-02-12 18:55:07.000000000 +0100 @@ -870,8 +870,8 @@ .RE .TP .BI fadvise_hint \fR=\fPstr -Use \fBposix_fadvise\fR\|(2) to advise the kernel what I/O patterns -are likely to be issued. Accepted values are: +Use \fBposix_fadvise\fR\|(2) or \fBposix_madvise\fR\|(2) to advise the kernel +what I/O patterns are likely to be issued. Accepted values are: .RS .RS .TP diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/fio.h new/fio-3.4/fio.h --- old/fio-3.3/fio.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/fio.h 2018-02-12 18:55:07.000000000 +0100 @@ -228,9 +228,9 @@ pid_t pid; char *orig_buffer; size_t orig_buffer_size; - volatile int terminate; volatile int runstate; - unsigned int last_was_sync; + volatile bool terminate; + bool last_was_sync; enum fio_ddir last_ddir; int mmapfd; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/fio_time.h new/fio-3.4/fio_time.h --- old/fio-3.3/fio_time.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/fio_time.h 2018-02-12 18:55:07.000000000 +0100 @@ -2,6 +2,7 @@ #define FIO_TIME_H #include <time.h> +#include <sys/time.h> #include "lib/types.h" struct thread_data; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/gclient.c new/fio-3.4/gclient.c --- old/fio-3.3/gclient.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/gclient.c 2018-02-12 18:55:07.000000000 +0100 @@ -298,6 +298,7 @@ client_ts.members++; client_ts.thread_number = p->ts.thread_number; client_ts.groupid = p->ts.groupid; + client_ts.sig_figs = p->ts.sig_figs; if (++sum_stat_nr == sum_stat_clients) { strcpy(client_ts.name, "All clients"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/init.c new/fio-3.4/init.c --- old/fio-3.3/init.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/init.c 2018-02-12 18:55:07.000000000 +0100 @@ -79,6 +79,7 @@ unsigned long fio_debug = 0; unsigned int fio_debug_jobno = -1; unsigned int *fio_debug_jobp = NULL; +unsigned int *fio_warned = NULL; static char cmd_optstr[256]; static bool did_arg; @@ -309,6 +310,7 @@ if (threads) { flow_exit(); fio_debug_jobp = NULL; + fio_warned = NULL; free_threads_shm(); } @@ -341,7 +343,7 @@ do { size_t size = max_jobs * sizeof(struct thread_data); - size += sizeof(unsigned int); + size += 2 * sizeof(unsigned int); #ifndef CONFIG_NO_SHM shm_id = shmget(0, size, IPC_CREAT | 0600); @@ -376,6 +378,8 @@ memset(threads, 0, max_jobs * sizeof(struct thread_data)); fio_debug_jobp = (unsigned int *)(threads + max_jobs); *fio_debug_jobp = -1; + fio_warned = fio_debug_jobp + 1; + *fio_warned = 0; flow_init(); @@ -802,11 +806,12 @@ o->verify_interval = o->min_bs[DDIR_READ]; /* - * Verify interval must be a factor or both min and max + * Verify interval must be a factor of both min and max * write size */ - if (o->verify_interval % o->min_bs[DDIR_WRITE] || - o->verify_interval % o->max_bs[DDIR_WRITE]) + if (!o->verify_interval || + (o->min_bs[DDIR_WRITE] % o->verify_interval) || + (o->max_bs[DDIR_WRITE] % o->verify_interval)) o->verify_interval = gcd(o->min_bs[DDIR_WRITE], o->max_bs[DDIR_WRITE]); } @@ -938,6 +943,11 @@ ret = 1; } + if (o->disable_lat) + o->lat_percentiles = 0; + if (o->disable_clat) + o->clat_percentiles = 0; + /* * Fix these up to be nsec internally */ @@ -1467,6 +1477,7 @@ td->ts.lat_percentiles = o->lat_percentiles; td->ts.percentile_precision = o->percentile_precision; memcpy(td->ts.percentile_list, o->percentile_list, sizeof(o->percentile_list)); + td->ts.sig_figs = o->sig_figs; for (i = 0; i < DDIR_RWDIR_CNT; i++) { td->ts.clat_stat[i].min_val = ULONG_MAX; @@ -1475,6 +1486,7 @@ td->ts.bw_stat[i].min_val = ULONG_MAX; td->ts.iops_stat[i].min_val = ULONG_MAX; } + td->ts.sync_stat.min_val = ULONG_MAX; td->ddir_seq_nr = o->ddir_seq_nr; if ((o->stonewall || o->new_group) && prev_group_jobs) { @@ -1578,7 +1590,7 @@ p.avg_msec = min(o->log_avg_msec, o->bw_avg_time); else o->bw_avg_time = p.avg_msec; - + p.hist_msec = o->log_hist_msec; p.hist_coarseness = o->log_hist_coarseness; @@ -1609,7 +1621,7 @@ p.avg_msec = min(o->log_avg_msec, o->iops_avg_time); else o->iops_avg_time = p.avg_msec; - + p.hist_msec = o->log_hist_msec; p.hist_coarseness = o->log_hist_coarseness; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/io_ddir.h new/fio-3.4/io_ddir.h --- old/fio-3.3/io_ddir.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/io_ddir.h 2018-02-12 18:55:07.000000000 +0100 @@ -5,13 +5,15 @@ DDIR_READ = 0, DDIR_WRITE = 1, DDIR_TRIM = 2, - DDIR_RWDIR_CNT = 3, DDIR_SYNC = 3, DDIR_DATASYNC, DDIR_SYNC_FILE_RANGE, DDIR_WAIT, DDIR_LAST, DDIR_INVAL = -1, + + DDIR_RWDIR_CNT = 3, + DDIR_RWDIR_SYNC_CNT = 4, }; static inline const char *io_ddir_name(enum fio_ddir ddir) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/io_u.c new/fio-3.4/io_u.c --- old/fio-3.3/io_u.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/io_u.c 2018-02-12 18:55:07.000000000 +0100 @@ -163,7 +163,6 @@ { struct zone_split_index *zsi; uint64_t lastb, send, stotal; - static int warned; unsigned int v; lastb = last_block(td, f, ddir); @@ -192,10 +191,8 @@ * Should never happen */ if (send == -1U) { - if (!warned) { + if (!fio_did_warn(FIO_WARN_ZONED_BUG)) log_err("fio: bug in zoned generation\n"); - warned = 1; - } goto bail; } else if (send > lastb) { /* @@ -223,7 +220,6 @@ { unsigned int v, send, stotal; uint64_t offset, lastb; - static int warned; struct zone_split_index *zsi; lastb = last_block(td, f, ddir); @@ -248,10 +244,8 @@ * Should never happen */ if (send == -1U) { - if (!warned) { + if (!fio_did_warn(FIO_WARN_ZONED_BUG)) log_err("fio: bug in zoned generation\n"); - warned = 1; - } goto bail; } @@ -922,6 +916,45 @@ *io_u = NULL; } +static void __fill_io_u_zone(struct thread_data *td, struct io_u *io_u) +{ + struct fio_file *f = io_u->file; + + /* + * See if it's time to switch to a new zone + */ + if (td->zone_bytes >= td->o.zone_size && td->o.zone_skip) { + td->zone_bytes = 0; + f->file_offset += td->o.zone_range + td->o.zone_skip; + + /* + * Wrap from the beginning, if we exceed the file size + */ + if (f->file_offset >= f->real_file_size) + f->file_offset = f->real_file_size - f->file_offset; + f->last_pos[io_u->ddir] = f->file_offset; + td->io_skip_bytes += td->o.zone_skip; + } + + /* + * If zone_size > zone_range, then maintain the same zone until + * zone_bytes >= zone_size. + */ + if (f->last_pos[io_u->ddir] >= (f->file_offset + td->o.zone_range)) { + dprint(FD_IO, "io_u maintain zone offset=%" PRIu64 "/last_pos=%" PRIu64 "\n", + f->file_offset, f->last_pos[io_u->ddir]); + f->last_pos[io_u->ddir] = f->file_offset; + } + + /* + * For random: if 'norandommap' is not set and zone_size > zone_range, + * map needs to be reset as it's done with zone_range everytime. + */ + if ((td->zone_bytes % td->o.zone_range) == 0) { + fio_file_reset(td, f); + } +} + static int fill_io_u(struct thread_data *td, struct io_u *io_u) { unsigned int is_random; @@ -938,21 +971,10 @@ goto out; /* - * See if it's time to switch to a new zone + * When file is zoned zone_range is always positive */ - if (td->zone_bytes >= td->o.zone_size && td->o.zone_skip) { - struct fio_file *f = io_u->file; - - td->zone_bytes = 0; - f->file_offset += td->o.zone_range + td->o.zone_skip; - - /* - * Wrap from the beginning, if we exceed the file size - */ - if (f->file_offset >= f->real_file_size) - f->file_offset = f->real_file_size - f->file_offset; - f->last_pos[io_u->ddir] = f->file_offset; - td->io_skip_bytes += td->o.zone_skip; + if (td->o.zone_range) { + __fill_io_u_zone(td, io_u); } /* @@ -971,9 +993,8 @@ } if (io_u->offset + io_u->buflen > io_u->file->real_file_size) { - dprint(FD_IO, "io_u %p, offset + buflen exceeds file size\n", - io_u); - dprint(FD_IO, " offset=%llu/buflen=%lu > %llu\n", + dprint(FD_IO, "io_u %p, off=0x%llx + len=0x%lx exceeds file size=0x%llx\n", + io_u, (unsigned long long) io_u->offset, io_u->buflen, (unsigned long long) io_u->file->real_file_size); return 1; @@ -986,7 +1007,7 @@ mark_random_map(td, io_u); out: - dprint_io_u(io_u, "fill_io_u"); + dprint_io_u(io_u, "fill"); td->zone_bytes += io_u->buflen; return 0; } @@ -1894,7 +1915,8 @@ if (no_reduce && per_unit_log(td->iops_log)) add_iops_sample(td, io_u, bytes); - } + } else if (ddir_sync(idx) && !td->o.disable_clat) + add_sync_clat_sample(&td->ts, llnsec); if (td->ts.nr_block_infos && io_u->ddir == DDIR_TRIM) { uint32_t *info = io_u_block_info(td, io_u); @@ -1932,6 +1954,12 @@ f->last_write_idx = 0; } +static bool should_account(struct thread_data *td) +{ + return ramp_time_over(td) && (td->runstate == TD_RUNNING || + td->runstate == TD_VERIFYING); +} + static void io_completed(struct thread_data *td, struct io_u **io_u_ptr, struct io_completion_data *icd) { @@ -1939,7 +1967,7 @@ enum fio_ddir ddir = io_u->ddir; struct fio_file *f = io_u->file; - dprint_io_u(io_u, "io complete"); + dprint_io_u(io_u, "complete"); assert(io_u->flags & IO_U_F_FLIGHT); io_u_clear(td, io_u, IO_U_F_FLIGHT | IO_U_F_BUSY_OK); @@ -1960,15 +1988,17 @@ } if (ddir_sync(ddir)) { - td->last_was_sync = 1; + td->last_was_sync = true; if (f) { f->first_write = -1ULL; f->last_write = -1ULL; } + if (should_account(td)) + account_io_completion(td, io_u, icd, ddir, io_u->buflen); return; } - td->last_was_sync = 0; + td->last_was_sync = false; td->last_ddir = ddir; if (!io_u->error && ddir_rw(ddir)) { @@ -1986,8 +2016,7 @@ if (ddir == DDIR_WRITE) file_log_write_comp(td, f, io_u->offset, bytes); - if (ramp_time_over(td) && (td->runstate == TD_RUNNING || - td->runstate == TD_VERIFYING)) + if (should_account(td)) account_io_completion(td, io_u, icd, ddir, bytes); icd->bytes_done[ddir] += bytes; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/io_u.h new/fio-3.4/io_u.h --- old/fio-3.3/io_u.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/io_u.h 2018-02-12 18:55:07.000000000 +0100 @@ -152,12 +152,17 @@ { struct fio_file *f = io_u->file; - dprint(FD_IO, "%s: io_u %p: off=%llu/len=%lu/ddir=%d", p, io_u, - (unsigned long long) io_u->offset, - io_u->buflen, io_u->ddir); if (f) - dprint(FD_IO, "/%s", f->file_name); - dprint(FD_IO, "\n"); + dprint(FD_IO, "%s: io_u %p: off=0x%llx,len=0x%lx,ddir=%d,file=%s\n", + p, io_u, + (unsigned long long) io_u->offset, + io_u->buflen, io_u->ddir, + f->file_name); + else + dprint(FD_IO, "%s: io_u %p: off=0x%llx,len=0x%lx,ddir=%d\n", + p, io_u, + (unsigned long long) io_u->offset, + io_u->buflen, io_u->ddir); } #else #define dprint_io_u(io_u, p) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/ioengines.c new/fio-3.4/ioengines.c --- old/fio-3.3/ioengines.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/ioengines.c 2018-02-12 18:55:07.000000000 +0100 @@ -224,7 +224,8 @@ if (td->io_ops->prep) { int ret = td->io_ops->prep(td, io_u); - dprint(FD_IO, "->prep(%p)=%d\n", io_u, ret); + dprint(FD_IO, "prep: io_u %p: ret=%d\n", io_u, ret); + if (ret) unlock_file(td, io_u->file); return ret; @@ -356,7 +357,7 @@ } if (ret == FIO_Q_COMPLETED) { - if (ddir_rw(io_u->ddir)) { + if (ddir_rw(io_u->ddir) || ddir_sync(io_u->ddir)) { io_u_mark_depth(td, 1); td->ts.total_io_u[io_u->ddir]++; } @@ -365,7 +366,7 @@ td->io_u_queued++; - if (ddir_rw(io_u->ddir)) + if (ddir_rw(io_u->ddir) || ddir_sync(io_u->ddir)) td->ts.total_io_u[io_u->ddir]++; if (td->io_u_queued >= td->o.iodepth_batch) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/iolog.c new/fio-3.4/iolog.c --- old/fio-3.3/iolog.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/iolog.c 2018-02-12 18:55:07.000000000 +0100 @@ -1141,8 +1141,6 @@ #ifdef CONFIG_ZLIB -static bool warned_on_drop; - static void iolog_put_deferred(struct io_log *log, void *ptr) { if (!ptr) @@ -1152,10 +1150,8 @@ if (log->deferred < IOLOG_MAX_DEFER) { log->deferred_items[log->deferred] = ptr; log->deferred++; - } else if (!warned_on_drop) { + } else if (!fio_did_warn(FIO_WARN_IOLOG_DROP)) log_err("fio: had to drop log entry free\n"); - warned_on_drop = true; - } pthread_mutex_unlock(&log->deferred_free_lock); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/libfio.c new/fio-3.4/libfio.c --- old/fio-3.3/libfio.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/libfio.c 2018-02-12 18:55:07.000000000 +0100 @@ -98,7 +98,7 @@ td->zone_bytes = 0; - td->last_was_sync = 0; + td->last_was_sync = false; td->rwmix_issues = 0; /* @@ -230,7 +230,7 @@ { fio_gettime(&td->terminate_time, NULL); write_barrier(); - td->terminate = 1; + td->terminate = true; } void fio_terminate_threads(unsigned int group_id) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/log.c new/fio-3.4/log.c --- old/fio-3.3/log.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/log.c 2018-02-12 18:55:07.000000000 +0100 @@ -36,6 +36,8 @@ do { *buffer = calloc(1, cur); + if (!*buffer) + return 0; va_copy(args, src_args); len = vsnprintf(*buffer, cur, fmt, args); @@ -51,6 +53,33 @@ return len; } +/* allocate buffer, fill with prefix string followed by vararg string */ +static size_t prevalist_to_buf(char **buffer, const char *pre, int prelen, + const char *fmt, va_list src_args) +{ + size_t len, cur = LOG_START_SZ; + va_list args; + + do { + *buffer = calloc(1, cur); + if (!*buffer) + return 0; + + va_copy(args, src_args); + memcpy(*buffer, pre, prelen); + len = prelen + vsnprintf(*buffer + prelen, cur - prelen, fmt, args); + va_end(args); + + if (len < cur) + break; + + cur = len + 1; + free(*buffer); + } while (1); + + return len; +} + size_t log_valist(const char *fmt, va_list args) { char *buffer; @@ -63,6 +92,28 @@ return len; } +/* add prefix for the specified type in front of the valist */ +void log_prevalist(int type, const char *fmt, va_list args) +{ + char pre[32]; + char *buffer; + size_t len; + int prelen; + pid_t pid; + + pid = gettid(); + if (fio_debug_jobp && *fio_debug_jobp != -1U + && pid != *fio_debug_jobp) + return; + + prelen = snprintf(pre, sizeof pre, "%-8s %-5u ", debug_levels[type].name, (int) pid); + if (prelen > 0) { + len = prevalist_to_buf(&buffer, pre, prelen, fmt, args); + len = log_info_buf(buffer, len); + free(buffer); + } +} + size_t log_info(const char *format, ...) { va_list args; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/log.h new/fio-3.4/log.h --- old/fio-3.3/log.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/log.h 2018-02-12 18:55:07.000000000 +0100 @@ -13,6 +13,7 @@ extern size_t log_info(const char *format, ...) __attribute__ ((__format__ (__printf__, 1, 2))); extern size_t __log_buf(struct buf_output *, const char *format, ...) __attribute__ ((__format__ (__printf__, 2, 3))); extern size_t log_valist(const char *str, va_list); +extern void log_prevalist(int type, const char *str, va_list); extern size_t log_info_buf(const char *buf, size_t len); extern int log_info_flush(void); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/mutex.c new/fio-3.4/mutex.c --- old/fio-3.3/mutex.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/mutex.c 2018-02-12 18:55:07.000000000 +0100 @@ -156,14 +156,15 @@ int fio_mutex_down_timeout(struct fio_mutex *mutex, unsigned int msecs) { struct timeval tv_s; + struct timespec base; struct timespec t; int ret = 0; assert(mutex->magic == FIO_MUTEX_MAGIC); gettimeofday(&tv_s, NULL); - t.tv_sec = tv_s.tv_sec; - t.tv_nsec = tv_s.tv_usec * 1000; + base.tv_sec = t.tv_sec = tv_s.tv_sec; + base.tv_nsec = t.tv_nsec = tv_s.tv_usec * 1000; t.tv_sec += msecs / 1000; t.tv_nsec += ((msecs * 1000000ULL) % 1000000000); @@ -181,7 +182,7 @@ * way too early, double check. */ ret = pthread_cond_timedwait(&mutex->cond, &mutex->lock, &t); - if (ret == ETIMEDOUT && !mutex_timed_out(&t, msecs)) + if (ret == ETIMEDOUT && !mutex_timed_out(&base, msecs)) ret = 0; } mutex->waiters--; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/options.c new/fio-3.4/options.c --- old/fio-3.3/options.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/options.c 2018-02-12 18:55:07.000000000 +0100 @@ -2443,7 +2443,7 @@ .posval = { { .ival = "0", .oval = F_ADV_NONE, - .help = "Don't issue fadvise", + .help = "Don't issue fadvise/madvise", }, { .ival = "1", .oval = F_ADV_TYPE, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/server.c new/fio-3.4/server.c --- old/fio-3.3/server.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/server.c 2018-02-12 18:55:07.000000000 +0100 @@ -1443,6 +1443,7 @@ dst->unit_base = cpu_to_le32(src->unit_base); dst->groupid = cpu_to_le32(src->groupid); dst->unified_rw_rep = cpu_to_le32(src->unified_rw_rep); + dst->sig_figs = cpu_to_le32(src->sig_figs); } /* diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/server.h new/fio-3.4/server.h --- old/fio-3.3/server.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/server.h 2018-02-12 18:55:07.000000000 +0100 @@ -49,7 +49,7 @@ }; enum { - FIO_SERVER_VER = 69, + FIO_SERVER_VER = 70, FIO_SERVER_MAX_FRAGMENT_PDU = 1024, FIO_SERVER_MAX_CMD_MB = 2048, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/stat.c new/fio-3.4/stat.c --- old/fio-3.3/stat.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/stat.c 2018-02-12 18:55:07.000000000 +0100 @@ -200,18 +200,17 @@ */ static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr, fio_fp64_t *plist, unsigned int precision, - bool is_clat, struct buf_output *out) + const char *pre, struct buf_output *out) { unsigned int divisor, len, i, j = 0; unsigned long long minv, maxv; unsigned long long *ovals; int per_line, scale_down, time_width; - const char *pre = is_clat ? "clat" : " lat"; bool is_last; char fmt[32]; len = calc_clat_percentiles(io_u_plat, nr, plist, &ovals, &maxv, &minv); - if (!len) + if (!len || !ovals) goto out; /* @@ -419,13 +418,26 @@ static void show_ddir_status(struct group_run_stats *rs, struct thread_stat *ts, int ddir, struct buf_output *out) { - const char *str[] = { " read", "write", " trim" }; + const char *str[] = { " read", "write", " trim", "sync" }; unsigned long runt; unsigned long long min, max, bw, iops; double mean, dev; char *io_p, *bw_p, *bw_p_alt, *iops_p; int i2p; + if (ddir_sync(ddir)) { + if (calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) { + log_buf(out, " %s:\n", "fsync/fdatasync/sync_file_range"); + display_lat(str[ddir], min, max, mean, dev, out); + show_clat_percentiles(ts->io_u_sync_plat, + ts->sync_stat.samples, + ts->percentile_list, + ts->percentile_precision, + str[ddir], out); + } + return; + } + assert(ddir_rw(ddir)); if (!ts->runtime[ddir]) @@ -460,11 +472,18 @@ display_lat(" lat", min, max, mean, dev, out); if (ts->clat_percentiles || ts->lat_percentiles) { + const char *name = ts->clat_percentiles ? "clat" : " lat"; + uint64_t samples; + + if (ts->clat_percentiles) + samples = ts->clat_stat[ddir].samples; + else + samples = ts->lat_stat[ddir].samples; + show_clat_percentiles(ts->io_u_plat[ddir], - ts->clat_stat[ddir].samples, + samples, ts->percentile_list, - ts->percentile_precision, - ts->clat_percentiles, out); + ts->percentile_precision, name, out); } if (calc_lat(&ts->bw_stat[ddir], &min, &max, &mean, &dev)) { double p_of_agg = 100.0, fkb_base = (double)rs->kb_base; @@ -796,6 +815,9 @@ show_latencies(ts, out); + if (ts->sync_stat.samples) + show_ddir_status(rs, ts, DDIR_SYNC, out); + runtime = ts->total_run_time; if (runtime) { double runt = (double) runtime; @@ -832,12 +854,13 @@ io_u_dist[1], io_u_dist[2], io_u_dist[3], io_u_dist[4], io_u_dist[5], io_u_dist[6]); - log_buf(out, " issued rwt: total=%llu,%llu,%llu," - " short=%llu,%llu,%llu," - " dropped=%llu,%llu,%llu\n", + log_buf(out, " issued rwts: total=%llu,%llu,%llu,%llu" + " short=%llu,%llu,%llu,0" + " dropped=%llu,%llu,%llu,0\n", (unsigned long long) ts->total_io_u[0], (unsigned long long) ts->total_io_u[1], (unsigned long long) ts->total_io_u[2], + (unsigned long long) ts->total_io_u[3], (unsigned long long) ts->short_io_u[0], (unsigned long long) ts->short_io_u[1], (unsigned long long) ts->short_io_u[2], @@ -961,12 +984,12 @@ double mean, dev, iops; unsigned int len; int i; - const char *ddirname[] = {"read", "write", "trim"}; + const char *ddirname[] = { "read", "write", "trim", "sync" }; struct json_object *dir_object, *tmp_object, *percentile_object, *clat_bins_object = NULL; char buf[120]; double p_of_agg = 100.0; - assert(ddir_rw(ddir)); + assert(ddir_rw(ddir) || ddir_sync(ddir)); if (ts->unified_rw_rep && ddir != DDIR_READ) return; @@ -975,64 +998,85 @@ json_object_add_value_object(parent, ts->unified_rw_rep ? "mixed" : ddirname[ddir], dir_object); - bw_bytes = 0; - bw = 0; - iops = 0.0; - if (ts->runtime[ddir]) { - uint64_t runt = ts->runtime[ddir]; - - bw_bytes = ((1000 * ts->io_bytes[ddir]) / runt); /* Bytes/s */ - bw = bw_bytes / 1024; /* KiB/s */ - iops = (1000.0 * (uint64_t) ts->total_io_u[ddir]) / runt; - } - - json_object_add_value_int(dir_object, "io_bytes", ts->io_bytes[ddir]); - json_object_add_value_int(dir_object, "io_kbytes", ts->io_bytes[ddir] >> 10); - json_object_add_value_int(dir_object, "bw_bytes", bw_bytes); - json_object_add_value_int(dir_object, "bw", bw); - json_object_add_value_float(dir_object, "iops", iops); - json_object_add_value_int(dir_object, "runtime", ts->runtime[ddir]); - json_object_add_value_int(dir_object, "total_ios", ts->total_io_u[ddir]); - json_object_add_value_int(dir_object, "short_ios", ts->short_io_u[ddir]); - json_object_add_value_int(dir_object, "drop_ios", ts->drop_io_u[ddir]); + if (ddir_rw(ddir)) { + bw_bytes = 0; + bw = 0; + iops = 0.0; + if (ts->runtime[ddir]) { + uint64_t runt = ts->runtime[ddir]; + + bw_bytes = ((1000 * ts->io_bytes[ddir]) / runt); /* Bytes/s */ + bw = bw_bytes / 1024; /* KiB/s */ + iops = (1000.0 * (uint64_t) ts->total_io_u[ddir]) / runt; + } + + json_object_add_value_int(dir_object, "io_bytes", ts->io_bytes[ddir]); + json_object_add_value_int(dir_object, "io_kbytes", ts->io_bytes[ddir] >> 10); + json_object_add_value_int(dir_object, "bw_bytes", bw_bytes); + json_object_add_value_int(dir_object, "bw", bw); + json_object_add_value_float(dir_object, "iops", iops); + json_object_add_value_int(dir_object, "runtime", ts->runtime[ddir]); + json_object_add_value_int(dir_object, "total_ios", ts->total_io_u[ddir]); + json_object_add_value_int(dir_object, "short_ios", ts->short_io_u[ddir]); + json_object_add_value_int(dir_object, "drop_ios", ts->drop_io_u[ddir]); + + if (!calc_lat(&ts->slat_stat[ddir], &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } + tmp_object = json_create_object(); + json_object_add_value_object(dir_object, "slat_ns", tmp_object); + json_object_add_value_int(tmp_object, "min", min); + json_object_add_value_int(tmp_object, "max", max); + json_object_add_value_float(tmp_object, "mean", mean); + json_object_add_value_float(tmp_object, "stddev", dev); + + if (!calc_lat(&ts->clat_stat[ddir], &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } + tmp_object = json_create_object(); + json_object_add_value_object(dir_object, "clat_ns", tmp_object); + json_object_add_value_int(tmp_object, "min", min); + json_object_add_value_int(tmp_object, "max", max); + json_object_add_value_float(tmp_object, "mean", mean); + json_object_add_value_float(tmp_object, "stddev", dev); + } else { + if (!calc_lat(&ts->sync_stat, &min, &max, &mean, &dev)) { + min = max = 0; + mean = dev = 0.0; + } - if (!calc_lat(&ts->slat_stat[ddir], &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; + tmp_object = json_create_object(); + json_object_add_value_object(dir_object, "lat_ns", tmp_object); + json_object_add_value_int(dir_object, "total_ios", ts->total_io_u[DDIR_SYNC]); + json_object_add_value_int(tmp_object, "min", min); + json_object_add_value_int(tmp_object, "max", max); + json_object_add_value_float(tmp_object, "mean", mean); + json_object_add_value_float(tmp_object, "stddev", dev); } - tmp_object = json_create_object(); - json_object_add_value_object(dir_object, "slat_ns", tmp_object); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); - - if (!calc_lat(&ts->clat_stat[ddir], &min, &max, &mean, &dev)) { - min = max = 0; - mean = dev = 0.0; - } - tmp_object = json_create_object(); - json_object_add_value_object(dir_object, "clat_ns", tmp_object); - json_object_add_value_int(tmp_object, "min", min); - json_object_add_value_int(tmp_object, "max", max); - json_object_add_value_float(tmp_object, "mean", mean); - json_object_add_value_float(tmp_object, "stddev", dev); if (ts->clat_percentiles || ts->lat_percentiles) { - len = calc_clat_percentiles(ts->io_u_plat[ddir], + if (ddir_rw(ddir)) { + len = calc_clat_percentiles(ts->io_u_plat[ddir], ts->clat_stat[ddir].samples, ts->percentile_list, &ovals, &maxv, &minv); + } else { + len = calc_clat_percentiles(ts->io_u_sync_plat, + ts->sync_stat.samples, + ts->percentile_list, &ovals, &maxv, + &minv); + } + + if (len > FIO_IO_U_LIST_MAX_LEN) + len = FIO_IO_U_LIST_MAX_LEN; } else len = 0; percentile_object = json_create_object(); json_object_add_value_object(tmp_object, "percentile", percentile_object); - for (i = 0; i < FIO_IO_U_LIST_MAX_LEN; i++) { - if (i >= len) { - json_object_add_value_int(percentile_object, "0.00", 0); - continue; - } + for (i = 0; i < len; i++) { snprintf(buf, sizeof(buf), "%f", ts->percentile_list[i].u.f); json_object_add_value_int(percentile_object, (const char *)buf, ovals[i]); } @@ -1043,13 +1087,23 @@ json_object_add_value_object(tmp_object, "bins", clat_bins_object); for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { - if (ts->io_u_plat[ddir][i]) { - snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); - json_object_add_value_int(clat_bins_object, (const char *)buf, ts->io_u_plat[ddir][i]); + if (ddir_rw(ddir)) { + if (ts->io_u_plat[ddir][i]) { + snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); + json_object_add_value_int(clat_bins_object, (const char *)buf, ts->io_u_plat[ddir][i]); + } + } else { + if (ts->io_u_sync_plat[i]) { + snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); + json_object_add_value_int(clat_bins_object, (const char *)buf, ts->io_u_sync_plat[i]); + } } } } + if (!ddir_rw(ddir)) + return; + if (!calc_lat(&ts->lat_stat[ddir], &min, &max, &mean, &dev)) { min = max = 0; mean = dev = 0.0; @@ -1172,7 +1226,7 @@ } static void json_add_job_opts(struct json_object *root, const char *name, - struct flist_head *opt_list, bool num_jobs) + struct flist_head *opt_list) { struct json_object *dir_object; struct flist_head *entry; @@ -1188,8 +1242,6 @@ const char *pos = ""; p = flist_entry(entry, struct print_option, list); - if (!num_jobs && !strcmp(p->name, "numjobs")) - continue; if (p->value) pos = p->value; json_object_add_value_string(dir_object, p->name, pos); @@ -1223,11 +1275,12 @@ } if (opt_list) - json_add_job_opts(root, "job options", opt_list, true); + json_add_job_opts(root, "job options", opt_list); add_ddir_status_json(ts, rs, DDIR_READ, root); add_ddir_status_json(ts, rs, DDIR_WRITE, root); add_ddir_status_json(ts, rs, DDIR_TRIM, root); + add_ddir_status_json(ts, rs, DDIR_SYNC, root); /* CPU Usage */ if (ts->total_run_time) { @@ -1483,6 +1536,8 @@ dst->kb_base = src->kb_base; if (!dst->unit_base) dst->unit_base = src->unit_base; + if (!dst->sig_figs) + dst->sig_figs = src->sig_figs; } void sum_thread_stats(struct thread_stat *dst, struct thread_stat *src, @@ -1522,24 +1577,25 @@ } } + sum_stat(&dst->sync_stat, &src->sync_stat, first); dst->usr_time += src->usr_time; dst->sys_time += src->sys_time; dst->ctx += src->ctx; dst->majf += src->majf; dst->minf += src->minf; - for (k = 0; k < FIO_IO_U_MAP_NR; k++) + for (k = 0; k < FIO_IO_U_MAP_NR; k++) { dst->io_u_map[k] += src->io_u_map[k]; - for (k = 0; k < FIO_IO_U_MAP_NR; k++) dst->io_u_submit[k] += src->io_u_submit[k]; - for (k = 0; k < FIO_IO_U_MAP_NR; k++) dst->io_u_complete[k] += src->io_u_complete[k]; - for (k = 0; k < FIO_IO_U_LAT_N_NR; k++) + } + for (k = 0; k < FIO_IO_U_LAT_N_NR; k++) { dst->io_u_lat_n[k] += src->io_u_lat_n[k]; - for (k = 0; k < FIO_IO_U_LAT_U_NR; k++) dst->io_u_lat_u[k] += src->io_u_lat_u[k]; - for (k = 0; k < FIO_IO_U_LAT_M_NR; k++) dst->io_u_lat_m[k] += src->io_u_lat_m[k]; + } + for (k = 0; k < FIO_IO_U_PLAT_NR; k++) + dst->io_u_sync_plat[k] += src->io_u_sync_plat[k]; for (k = 0; k < DDIR_RWDIR_CNT; k++) { if (!dst->unified_rw_rep) { @@ -1553,6 +1609,8 @@ } } + dst->total_io_u[DDIR_SYNC] += src->total_io_u[DDIR_SYNC]; + for (k = 0; k < DDIR_RWDIR_CNT; k++) { int m; @@ -1591,6 +1649,7 @@ ts->bw_stat[j].min_val = -1UL; ts->iops_stat[j].min_val = -1UL; } + ts->sync_stat.min_val = -1UL; ts->groupid = -1; } @@ -1817,7 +1876,7 @@ json_object_add_value_int(root, "timestamp_ms", ms_since_epoch); json_object_add_value_string(root, "time", time_buf); global = get_global_options(); - json_add_job_opts(root, "global options", &global->opt_list, false); + json_add_job_opts(root, "global options", &global->opt_list); array = json_create_array(); json_object_add_value_array(root, "jobs", array); } @@ -2224,10 +2283,15 @@ ts->short_io_u[i] = 0; ts->drop_io_u[i] = 0; - for (j = 0; j < FIO_IO_U_PLAT_NR; j++) + for (j = 0; j < FIO_IO_U_PLAT_NR; j++) { ts->io_u_plat[i][j] = 0; + if (!i) + ts->io_u_sync_plat[j] = 0; + } } + ts->total_io_u[DDIR_SYNC] = 0; + for (i = 0; i < FIO_IO_U_MAP_NR; i++) { ts->io_u_map[i] = 0; ts->io_u_submit[i] = 0; @@ -2350,6 +2414,15 @@ __add_log_sample(iolog, data, ddir, bs, mtime_since_genesis(), 0); } +void add_sync_clat_sample(struct thread_stat *ts, unsigned long long nsec) +{ + unsigned int idx = plat_val_to_idx(nsec); + assert(idx < FIO_IO_U_PLAT_NR); + + ts->io_u_sync_plat[idx]++; + add_stat_sample(&ts->sync_stat, nsec); +} + static void add_clat_percentile_sample(struct thread_stat *ts, unsigned long long nsec, enum fio_ddir ddir) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/stat.h new/fio-3.4/stat.h --- old/fio-3.3/stat.h 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/stat.h 2018-02-12 18:55:07.000000000 +0100 @@ -159,6 +159,7 @@ /* * bandwidth and latency stats */ + struct io_stat sync_stat __attribute__((aligned(8)));/* fsync etc stats */ struct io_stat clat_stat[DDIR_RWDIR_CNT]; /* completion latency */ struct io_stat slat_stat[DDIR_RWDIR_CNT]; /* submission latency */ struct io_stat lat_stat[DDIR_RWDIR_CNT]; /* total latency */ @@ -188,9 +189,10 @@ uint32_t io_u_lat_u[FIO_IO_U_LAT_U_NR]; uint32_t io_u_lat_m[FIO_IO_U_LAT_M_NR]; uint32_t io_u_plat[DDIR_RWDIR_CNT][FIO_IO_U_PLAT_NR]; + uint32_t io_u_sync_plat[FIO_IO_U_PLAT_NR]; uint32_t pad; - uint64_t total_io_u[DDIR_RWDIR_CNT]; + uint64_t total_io_u[DDIR_RWDIR_SYNC_CNT]; uint64_t short_io_u[DDIR_RWDIR_CNT]; uint64_t drop_io_u[DDIR_RWDIR_CNT]; uint64_t total_submit; @@ -318,6 +320,8 @@ unsigned int); extern void add_bw_sample(struct thread_data *, struct io_u *, unsigned int, unsigned long long); +extern void add_sync_clat_sample(struct thread_stat *ts, + unsigned long long nsec); extern int calc_log_samples(void); extern struct io_log *agg_io_log[DDIR_RWDIR_CNT]; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/tools/hist/fiologparser_hist.py new/fio-3.4/tools/hist/fiologparser_hist.py --- old/fio-3.3/tools/hist/fiologparser_hist.py 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/tools/hist/fiologparser_hist.py 2018-02-12 18:55:07.000000000 +0100 @@ -177,7 +177,7 @@ avg = weighted_average(vs, ws) values = [mn, avg] + list(ps) + [mx] - row = [end, ss_cnt] + map(lambda x: float(x) / ctx.divisor, values) + row = [end, ss_cnt] + [float(x) / ctx.divisor for x in values] fmt = "%d, %d, %d, " + fmt_float_list(ctx, 5) + ", %d" print (fmt % tuple(row)) @@ -288,9 +288,9 @@ max_cols = guess_max_from_bins(ctx, __HIST_COLUMNS) coarseness = int(np.log2(float(max_cols) / __HIST_COLUMNS)) - bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness), np.arange(__HIST_COLUMNS)), dtype=float) - lower_bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness, 0.0), np.arange(__HIST_COLUMNS)), dtype=float) - upper_bin_vals = np.array(map(lambda x: plat_idx_to_val_coarse(x, coarseness, 1.0), np.arange(__HIST_COLUMNS)), dtype=float) + bin_vals = np.array([plat_idx_to_val_coarse(x, coarseness) for x in np.arange(__HIST_COLUMNS)], dtype=float) + lower_bin_vals = np.array([plat_idx_to_val_coarse(x, coarseness, 0.0) for x in np.arange(__HIST_COLUMNS)], dtype=float) + upper_bin_vals = np.array([plat_idx_to_val_coarse(x, coarseness, 1.0) for x in np.arange(__HIST_COLUMNS)], dtype=float) fps = [open(f, 'r') for f in ctx.FILE] gen = histogram_generator(ctx, fps, ctx.buff_size) @@ -333,7 +333,8 @@ start += ctx.interval end = start + ctx.interval finally: - map(lambda f: f.close(), fps) + for fp in fps: + fp.close() if __name__ == '__main__': diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/fio-3.3/verify.c new/fio-3.4/verify.c --- old/fio-3.3/verify.c 2017-12-19 21:16:36.000000000 +0100 +++ new/fio-3.4/verify.c 2018-02-12 18:55:07.000000000 +0100 @@ -87,8 +87,13 @@ { unsigned int hdr_inc; + /* + * If we use bs_unaligned, buflen can be larger than the verify + * interval (which just defaults to the smallest blocksize possible). + */ hdr_inc = io_u->buflen; - if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen) + if (td->o.verify_interval && td->o.verify_interval <= io_u->buflen && + !td->o.bs_unaligned) hdr_inc = td->o.verify_interval; return hdr_inc; @@ -236,7 +241,6 @@ }; #define DUMP_BUF_SZ 255 -static int dump_buf_warned; static void dump_buf(char *buf, unsigned int len, unsigned long long offset, const char *type, struct fio_file *f) @@ -255,10 +259,8 @@ buf_left -= strlen(fname); if (buf_left <= 0) { - if (!dump_buf_warned) { + if (!fio_did_warn(FIO_WARN_VERIFY_BUF)) log_err("fio: verify failure dump buffer too small\n"); - dump_buf_warned = 1; - } free(ptr); return; } @@ -1175,7 +1177,6 @@ struct verify_header *hdr, unsigned int header_num, unsigned int header_len, uint64_t rand_seed) { - if (td->o.verify != VERIFY_PATTERN_NO_HDR) __fill_hdr(td, io_u, hdr, header_num, header_len, rand_seed); }
