Attached is a patch which makes the read/write buffer size configurable
on a per-file-descriptor basis, ie;
apr_file_open(&fh, "/dev/zero", APR_WRITE | APR_BUFFERED,
APR_UREAD | APR_UWRITE | APR_GREAD, p);
/* Increase the buffer size to 10k */
apr_file_set_buffer_size(fh, 10240);
/* Query the buffer size */
current_bufsize = apr_file_get_buffer_size(filetest);
It supports increasing and decreasing the buffer size, as well as making
an unbuffered file handle buffered, and a buffered file unbuffered.
The patch also adds a new subpool to the file handle, for the buffer
itself, with the subpool being destroyed on buffer resizes.
Other things I'm thinking about;
Destroying the subpool on apr_file_close, after all the lifetime
of the buffer should be exactly concurrent with the lifetime of
the file descriptor.
Maybe making something like;
apr_file_set_default_buffer_size(apr_off_t foo);
work.
The implementation in buffer.c should be portable, with the only
changing the file->thlock to file->mutex being neccessary for some of
the other architectures, though I havn't done that porting just yet, I
want to be able to test it on win32 at least first, and maybe get some
feedback :-)
Apart from my immediate problem of needing a buffered stdin, I've done
some extensive benchmarking in the past of the efficiency of various
buffer sizes;
http://www.stdlib.net/~colmmacc/Apachecon-EU2005/
See slides 18-21 of the presentation. There are definitely some major
real-world benefits to giving people the ability to tune the buffer
sizes.
--
Colm MacCárthaigh Public Key: [EMAIL PROTECTED]
Index: test/testfile.c
===================================================================
--- test/testfile.c (revision 279021)
+++ test/testfile.c (working copy)
@@ -326,6 +326,33 @@
apr_file_close(filetest);
}
+static void test_buffer_size_set_get(abts_case *tc, void *data)
+{
+ apr_status_t rv;
+ apr_file_t *filetest = NULL;
+
+ rv = apr_file_open(&filetest, FILENAME,
+ APR_WRITE | APR_BUFFERED,
+ APR_UREAD | APR_UWRITE | APR_GREAD, p);
+ ABTS_INT_EQUAL(tc, APR_SUCCESS, rv);
+
+ rv = apr_file_get_buffer_size(filetest);
+ ABTS_INT_EQUAL(tc, APR_BUFFERSIZE, rv);
+
+ rv = apr_file_set_buffer_size(filetest, 10240);
+ ABTS_INT_EQUAL(tc, APR_SUCCESS, rv);
+
+ rv = apr_file_get_buffer_size(filetest);
+ ABTS_INT_EQUAL(tc, 10240, rv);
+
+ rv = apr_file_set_buffer_size(filetest, 12);
+ ABTS_INT_EQUAL(tc, APR_SUCCESS, rv);
+
+ rv = apr_file_get_buffer_size(filetest);
+ ABTS_INT_EQUAL(tc, 12, rv);
+
+ apr_file_close(filetest);
+}
static void test_getc(abts_case *tc, void *data)
{
apr_file_t *f = NULL;
@@ -802,6 +829,7 @@
abts_run_test(suite, test_bigfprintf, NULL);
abts_run_test(suite, test_fail_write_flush, NULL);
abts_run_test(suite, test_fail_read_flush, NULL);
+ abts_run_test(suite, test_buffer_size_set_get, NULL);
return suite;
}
Index: include/apr_file_io.h
===================================================================
--- include/apr_file_io.h (revision 279021)
+++ include/apr_file_io.h (working copy)
@@ -325,6 +325,23 @@
apr_pool_t *pool);
/**
+ * Change the buffer size for an existing apr file pointer.
+ * @param thefile The apr file pointer to tune
+ * @param bufsize The new buffer size
+ *
+ * @remark Setting the bufsize to zero will disable buffering for the
+ * file.
+ */
+APR_DECLARE(apr_status_t) apr_file_set_buffer_size(apr_file_t *thefile,
+ apr_off_t bufsize);
+
+/**
+ * Return the buffer size for an existing apr file pointer.
+ * @param thefile The apr file pointer to query
+ */
+APR_DECLARE(apr_off_t) apr_file_get_buffer_size(apr_file_t *thefile);
+
+/**
* Read data from the specified file.
* @param thefile The file descriptor to read from.
* @param buf The buffer to store the data to.
Index: include/arch/win32/apr_arch_file_io.h
===================================================================
--- include/arch/win32/apr_arch_file_io.h (revision 279021)
+++ include/arch/win32/apr_arch_file_io.h (working copy)
@@ -172,8 +172,10 @@
/* Stuff for buffered mode */
char *buffer;
+ apr_pool_p *bufsubpool; // Subpool used for the buffer
apr_size_t bufpos; // Read/Write position in buffer
apr_size_t dataRead; // amount of valid data read into buffer
+ apr_size_t bufsize; // The size of the buffer
int direction; // buffer being used for 0 = read, 1 = write
apr_off_t filePtr; // position in file of handle
apr_thread_mutex_t *mutex; // mutex semaphore, must be owned to access the
above fields
Index: include/arch/os2/apr_arch_file_io.h
===================================================================
--- include/arch/os2/apr_arch_file_io.h (revision 279021)
+++ include/arch/os2/apr_arch_file_io.h (working copy)
@@ -48,7 +48,9 @@
/* Stuff for buffered mode */
char *buffer;
+ apr_pool_p *bufsubpool; // Subpool used for the buffer
int bufpos; // Read/Write position in buffer
+ apr_off_t bufsize; // The size of the buffer
unsigned long dataRead; // amount of valid data read into buffer
int direction; // buffer being used for 0 = read, 1 = write
unsigned long filePtr; // position in file of handle
Index: include/arch/unix/apr_arch_file_io.h
===================================================================
--- include/arch/unix/apr_arch_file_io.h (revision 279021)
+++ include/arch/unix/apr_arch_file_io.h (working copy)
@@ -101,9 +101,11 @@
#endif
/* Stuff for buffered mode */
char *buffer;
+ apr_pool_t *bufsubpool; /* Subpool for the buffer */
int bufpos; /* Read/Write position in buffer */
unsigned long dataRead; /* amount of valid data read into buffer */
int direction; /* buffer being used for 0 = read, 1 = write */
+ apr_off_t bufsize; /* The size of the buffer */
unsigned long filePtr; /* position in file of handle */
#if APR_HAS_THREADS
struct apr_thread_mutex_t *thlock;
Index: include/arch/netware/apr_arch_file_io.h
===================================================================
--- include/arch/netware/apr_arch_file_io.h (revision 279021)
+++ include/arch/netware/apr_arch_file_io.h (working copy)
@@ -95,7 +95,9 @@
/* Stuff for buffered mode */
char *buffer;
+ apr_pool_t *bufsubpool; /* Subpool for the buffer */
int bufpos; /* Read/Write position in buffer */
+ apr_off_t bufsize; /* The size of the buffer */
apr_off_t dataRead; /* amount of valid data read into buffer */
int direction; /* buffer being used for 0 = read, 1 = write */
apr_off_t filePtr; /* position in file of handle */
Index: file_io/win32/open.c
===================================================================
--- file_io/win32/open.c (revision 279021)
+++ file_io/win32/open.c (working copy)
@@ -387,6 +387,7 @@
if (flag & APR_BUFFERED) {
(*new)->buffered = 1;
(*new)->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ (*new)->bufsize = APR_FILE_BUFSIZE;
}
/* Need the mutex to handled buffered and O_APPEND style file i/o */
if ((*new)->buffered || (*new)->append) {
@@ -529,6 +530,7 @@
if (flags & APR_BUFFERED) {
(*file)->buffered = 1;
(*file)->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ (*file)->bufsize = APR_FILE_BUFSIZE;
}
if ((*file)->append || (*file)->buffered) {
Index: file_io/win32/readwrite.c
===================================================================
--- file_io/win32/readwrite.c (revision 279021)
+++ file_io/win32/readwrite.c (working copy)
@@ -177,7 +177,7 @@
if (thefile->bufpos >= thefile->dataRead) {
apr_size_t read;
rv = read_with_timeout(thefile, thefile->buffer,
- APR_FILE_BUFSIZE, &read);
+ thefile->bufsize, &read);
if (read == 0) {
if (rv == APR_EOF)
thefile->eof_hit = TRUE;
@@ -251,10 +251,11 @@
rv = 0;
while (rv == 0 && size > 0) {
- if (thefile->bufpos == APR_FILE_BUFSIZE) // write buffer is full
+ if (thefile->bufpos == thefile->bufsize) // write buffer is full
rv = apr_file_flush(thefile);
- blocksize = size > APR_FILE_BUFSIZE - thefile->bufpos ?
APR_FILE_BUFSIZE - thefile->bufpos : size;
+ blocksize = size > thefile->bufsize - thefile->bufpos ?
+ thefile->bufsize - thefile->bufpos : size;
memcpy(thefile->buffer + thefile->bufpos, pos, blocksize);
thefile->bufpos += blocksize;
pos += blocksize;
Index: file_io/win32/filedup.c
===================================================================
--- file_io/win32/filedup.c (revision 279021)
+++ file_io/win32/filedup.c (working copy)
@@ -145,7 +145,8 @@
memcpy(*new_file, old_file, sizeof(apr_file_t));
(*new_file)->pool = p;
if (old_file->buffered) {
- (*new_file)->buffer = apr_palloc(p, APR_FILE_BUFSIZE);
+ (*new_file)->buffer = apr_palloc(p, old_file->bufsize);
+ (*new_file)->bufsize = old_file->bufsize;
if (old_file->direction == 1) {
memcpy((*new_file)->buffer, old_file->buffer, old_file->bufpos);
}
Index: file_io/os2/open.c
===================================================================
--- file_io/os2/open.c (revision 279021)
+++ file_io/os2/open.c (working copy)
@@ -60,6 +60,7 @@
if (dafile->buffered) {
dafile->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ dafile->bufsize = APR_FILE_BUFSIZE;
rv = apr_thread_mutex_create(&dafile->mutex, 0, pool);
if (rv)
@@ -194,6 +195,7 @@
apr_status_t rv;
(*file)->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ (*file)->bufsize = APR_FILE_BUFSIZE;
rv = apr_thread_mutex_create(&(*file)->mutex, 0, pool);
if (rv)
Index: file_io/os2/readwrite.c
===================================================================
--- file_io/os2/readwrite.c (revision 279021)
+++ file_io/os2/readwrite.c (working copy)
@@ -52,7 +52,7 @@
if (thefile->bufpos >= thefile->dataRead) {
ULONG bytesread;
rc = DosRead(thefile->filedes, thefile->buffer,
- APR_FILE_BUFSIZE, &bytesread);
+ thefile->bufsize, &bytesread);
if (bytesread == 0) {
if (rc == 0)
@@ -143,10 +143,11 @@
}
while (rc == 0 && size > 0) {
- if (thefile->bufpos == APR_FILE_BUFSIZE) // write buffer is full
+ if (thefile->bufpos == thefile->bufsize) // write buffer is full
rc = apr_file_flush(thefile);
- blocksize = size > APR_FILE_BUFSIZE - thefile->bufpos ?
APR_FILE_BUFSIZE - thefile->bufpos : size;
+ blocksize = size > thefile->bufsize - thefile->bufpos ?
+ thefile->bufsize - thefile->bufpos : size;
memcpy(thefile->buffer + thefile->bufpos, pos, blocksize);
thefile->bufpos += blocksize;
pos += blocksize;
Index: file_io/os2/filedup.c
===================================================================
--- file_io/os2/filedup.c (revision 279021)
+++ file_io/os2/filedup.c (working copy)
@@ -91,7 +91,8 @@
(*new_file)->pool = p;
if (old_file->buffered) {
- (*new_file)->buffer = apr_palloc(p, APR_FILE_BUFSIZE);
+ (*new_file)->buffer = apr_palloc(p, old_file->bufsize);
+ (*new_file)->bufsize = old_file->bufsize;
if (old_file->direction == 1) {
memcpy((*new_file)->buffer, old_file->buffer, old_file->bufpos);
Index: file_io/unix/open.c
===================================================================
--- file_io/unix/open.c (revision 279021)
+++ file_io/unix/open.c (working copy)
@@ -150,7 +150,9 @@
(*new)->buffered = (flag & APR_BUFFERED) > 0;
if ((*new)->buffered) {
- (*new)->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ apr_pool_create(&(*new)->bufsubpool, pool);
+ (*new)->buffer = apr_palloc((*new)->bufsubpool, APR_FILE_BUFSIZE);
+ (*new)->bufsize = APR_FILE_BUFSIZE;
#if APR_HAS_THREADS
if ((*new)->flags & APR_XTHREAD) {
(*new)->thlock = thlock;
@@ -239,7 +241,9 @@
#endif
if ((*file)->buffered) {
- (*file)->buffer = apr_palloc(pool, APR_FILE_BUFSIZE);
+ apr_pool_create(&(*file)->bufsubpool, pool);
+ (*file)->buffer = apr_palloc((*file)->bufsubpool, APR_FILE_BUFSIZE);
+ (*file)->bufsize = APR_FILE_BUFSIZE;
#if APR_HAS_THREADS
if ((*file)->flags & APR_XTHREAD) {
apr_status_t rv;
Index: file_io/unix/buffer.c
===================================================================
--- file_io/unix/buffer.c (revision 0)
+++ file_io/unix/buffer.c (revision 0)
@@ -0,0 +1,98 @@
+/* Copyright 2000-2005 The Apache Software Foundation or its licensors, as
+ * applicable.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "apr_arch_file_io.h"
+#include "apr_pools.h"
+#include "apr_thread_mutex.h"
+
+APR_DECLARE(apr_status_t) apr_file_set_buffer_size(apr_file_t *file,
+ apr_off_t newbufsize)
+{
+ apr_status_t rv;
+
+#if APR_HAS_THREADS
+ if (file->thlock) {
+ apr_thread_mutex_lock(file->thlock);
+ }
+#endif
+
+ if(file->buffered) {
+ /* Flush the existing buffer */
+ rv = apr_file_flush(file);
+ if (rv != APR_SUCCESS) {
+#if APR_HAS_THREADS
+ if (file->thlock) {
+ apr_thread_mutex_lock(file->thlock);
+ }
+#endif
+ return rv;
+ }
+
+ file->buffer = NULL;
+ file->bufsize = 0;
+ file->buffered = 0;
+ file->bufpos = 0;
+ file->direction = 0;
+ file->dataRead = 0;
+
+ if (newbufsize == 0) {
+ /* Setting the buffer size to zero is equivalent to turning
+ * buffering off.
+ */
+#if APR_HAS_THREADS
+ if (file->thlock) {
+ apr_thread_mutex_unlock(file->thlock);
+ }
+#endif
+ return APR_SUCCESS;
+ }
+ else {
+ /* Destroy the existing pool */
+ apr_pool_destroy(file->bufsubpool);
+ }
+ }
+
+ if (newbufsize > 0) {
+ /* Create a new subpool, and the buffer */
+ rv = apr_pool_create(&file->bufsubpool, file->pool);
+ if (rv != APR_SUCCESS) {
+#if APR_HAS_THREADS
+ if (file->thlock) {
+ apr_thread_mutex_unlock(file->thlock);
+ }
+#endif
+ return rv;
+ }
+ file->buffer = apr_palloc(file->bufsubpool, newbufsize);
+ file->bufsize = newbufsize;
+ file->buffered = 1;
+ file->bufpos = 0;
+ file->direction = 0;
+ file->dataRead = 0;
+ }
+
+#if APR_HAS_THREADS
+ if (file->thlock) {
+ apr_thread_mutex_unlock(file->thlock);
+ }
+#endif
+ return APR_SUCCESS;
+}
+
+APR_DECLARE(apr_off_t) apr_file_get_buffer_size(apr_file_t *file)
+{
+ return file->bufsize;
+}
Index: file_io/unix/readwrite.c
===================================================================
--- file_io/unix/readwrite.c (revision 279021)
+++ file_io/unix/readwrite.c (working copy)
@@ -70,7 +70,7 @@
}
while (rv == 0 && size > 0) {
if (thefile->bufpos >= thefile->dataRead) {
- int bytesread = read(thefile->filedes, thefile->buffer,
APR_FILE_BUFSIZE);
+ int bytesread = read(thefile->filedes, thefile->buffer,
thefile->bufsize);
if (bytesread == 0) {
thefile->eof_hit = TRUE;
rv = APR_EOF;
@@ -177,11 +177,11 @@
rv = 0;
while (rv == 0 && size > 0) {
- if (thefile->bufpos == APR_FILE_BUFSIZE) /* write buffer is
full*/
+ if (thefile->bufpos == thefile->bufsize) /* write buffer is
full*/
rv = apr_file_flush(thefile);
- blocksize = size > APR_FILE_BUFSIZE - thefile->bufpos ?
- APR_FILE_BUFSIZE - thefile->bufpos : size;
+ blocksize = size > thefile->bufsize - thefile->bufpos ?
+ thefile->bufsize - thefile->bufpos : size;
memcpy(thefile->buffer + thefile->bufpos, pos, blocksize);
thefile->bufpos += blocksize;
pos += blocksize;
Index: file_io/unix/filedup.c
===================================================================
--- file_io/unix/filedup.c (revision 279021)
+++ file_io/unix/filedup.c (working copy)
@@ -62,7 +62,8 @@
* got one.
*/
if ((*new_file)->buffered && !(*new_file)->buffer) {
- (*new_file)->buffer = apr_palloc(p, APR_FILE_BUFSIZE);
+ (*new_file)->buffer = apr_palloc(p, old_file->bufsize);
+ (*new_file)->bufsize = old_file->bufsize;
}
/* this is the way dup() works */
@@ -121,7 +122,8 @@
memcpy(*new_file, old_file, sizeof(apr_file_t));
(*new_file)->pool = p;
if (old_file->buffered) {
- (*new_file)->buffer = apr_palloc(p, APR_FILE_BUFSIZE);
+ (*new_file)->buffer = apr_palloc(p, old_file->bufsize);
+ (*new_file)->bufsize = old_file->bufsize;
if (old_file->direction == 1) {
memcpy((*new_file)->buffer, old_file->buffer, old_file->bufpos);
}
