Heya, I've been enjoying this patch without any ill-effects the past
week while editing large audio files on 4-core machine.
It's currently on the "pu-starla" branch of git://bogomips.org/sox.git
I can make a patch formal submission on SF in the next few days
if you all like it :)
>From 71d405612c76b3ddeb560ca08a7e2f7c1bc01ade Mon Sep 17 00:00:00 2001
From: Eric Wong <normalper...@yhbt.net>
Date: Fri, 22 Jun 2012 08:34:25 +0000
Subject: [PATCH] speed up "|program" inputs on Linux 2.6.35+
Linux 2.6.35+ allows pipe buffer resizing via fcntl(2). When
running multi-threaded SoX invocations with large buffers, the
default pipe size (64K) can be too small and become a bottleneck
for IPC. Increasing the pipe to the maximum allowed size
reduces the amount of stalls in data flow between processes (SoX
or otherwise).
When using SOX_OPTS="--multi-thread --buffer 131072" on a 4-core
system, a command like:
sox -M "|sox $< -p $(lft_fx)" "|sox $< -p $(rgt_fx)" $@
..can run significantly faster (10-80%) depending on the
processing chain, file sizes/formats and effects in use.
Before this patch, using "--buffer 131072" could be hugely
detrimental to performance due to the pipe being only half
the size (64K) of the SoX buffer.
---
src/formats.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/src/formats.c b/src/formats.c
index 181a3e4..7a7b6fa 100644
--- a/src/formats.c
+++ b/src/formats.c
@@ -18,6 +18,7 @@
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#define _GNU_SOURCE
#include "sox_i.h"
#include <assert.h>
@@ -37,6 +38,10 @@
#include <magic.h>
#endif
+#ifdef HAVE_UNISTD_H
+# include <unistd.h>
+#endif
+
#define PIPE_AUTO_DETECT_SIZE 256 /* Only as much as we can rewind a pipe */
#define AUTO_DETECT_SIZE 4096 /* For seekable file, so no restriction */
@@ -366,6 +371,50 @@ static int xfclose(FILE * file, lsx_io_type io_type)
fclose(file);
}
+static void incr_pipe_size(FILE *f)
+{
+/*
+ * Linux 2.6.35 and later has the ability to expand the pipe buffer
+ * Try to get it as big as possible to avoid stalls when SoX itself
+ * is using big buffers
+ */
+#if defined(F_GETPIPE_SZ) && defined(F_SETPIPE_SZ)
+ static long max_pipe_size;
+
+ /* read the maximum size of the pipe the first time this is called */
+ if (max_pipe_size == 0) {
+ const char path[] = "/proc/sys/fs/pipe-max-size";
+ int fd = open(path, O_RDONLY);
+
+ max_pipe_size = -1;
+ if (fd >= 0) {
+ char buf[80];
+ ssize_t r = read(fd, buf, sizeof(buf));
+
+ if (r > 0) {
+ buf[r] = 0;
+ max_pipe_size = strtol(buf, NULL, 10);
+
+ /* guard against obviously wrong values on messed up systems */
+ if (max_pipe_size <= PIPE_BUF || max_pipe_size > INT_MAX)
+ max_pipe_size = -1;
+ }
+ close(fd);
+ }
+ }
+
+ if (max_pipe_size > PIPE_BUF) {
+ int fd = fileno(f);
+
+ if (fcntl(fd, F_SETPIPE_SZ, max_pipe_size) >= 0)
+ lsx_debug("got pipe %ld bytes\n", max_pipe_size);
+ else
+ lsx_warn("couldn't set pipe size to %ld bytes: %s\n",
+ max_pipe_size, strerror(errno));
+ }
+#endif /* do nothing for platforms without F_{GET,SET}PIPE_SZ */
+}
+
static FILE * xfopen(char const * identifier, char const * mode, lsx_io_type *
io_type)
{
*io_type = lsx_io_file;
@@ -378,6 +427,7 @@ static FILE * xfopen(char const * identifier, char const *
mode, lsx_io_type * i
#endif
f = popen(identifier + 1, POPEN_MODE);
*io_type = lsx_io_pipe;
+ incr_pipe_size(f);
#else
lsx_fail("this build of SoX cannot open pipes");
#endif
@@ -390,6 +440,7 @@ static FILE * xfopen(char const * identifier, char const *
mode, lsx_io_type * i
char * command = lsx_malloc(strlen(command_format) + strlen(identifier));
sprintf(command, command_format, identifier);
f = popen(command, POPEN_MODE);
+ incr_pipe_size(f);
free(command);
*io_type = lsx_io_url;
#else
--
1.7.11.rc3.1.gb9dc72f
------------------------------------------------------------------------------
Live Security Virtual Conference
Exclusive live event will cover all the ways today's security and
threat landscape has changed and how IT managers can respond. Discussions
will include endpoint security, mobile security and the latest in malware
threats. http://www.accelacomm.com/jaw/sfrnl04242012/114/50122263/
_______________________________________________
SoX-devel mailing list
SoX-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/sox-devel
