For whatever reason, part of the original patch was incorrect. Here's
the update against apr-util HEAD...
Again there is no storage/recovery of these page size parameters yet,
and we have yet to define an API for opening/creating an SDBM with
such alternatives. It should be obvious in the open call where I've hacked
in our default values, for anyone to experiment with.
Immediate beneficiaries include mod_ssl and dav, where pages may need
to be much larger than the default.
Bill
At 02:56 PM 9/19/2003, William A. Rowe, Jr. wrote:
>As a few here already know, we used the 'Standard' page size for our
>apr-util SDBM implementation. This assures that you can modify our
>sdbm files from perl and other tools.
>
>However, it's pretty clear that 1024 just doesn't cut it when it comes
>to huge objects from ssl caching and other sorts of large blobs. The
>modssl project (1.3) had tweaked sdbm to take larger page sizes, but
>this leads to an inflexible implementation (another fixed page size, but
>incompatible with other implementations.)
>
>I'd like to see apr-util support alternate sized dbm pages, but without
>making fixed assumptions about those sizes (other than this - the
>structure uses short's for offsets, so the page size does have a cap.)
>
>The attached patch allows the SDBM to be of any arbitrary data and
>directory page sizes. However, I've not hacked in any method of
>declaring what those sizes should be (they are the 'standard' defaults
>at the moment.)
>
>What do folks believe is the most rational approach to expanding support
>to create an sdbm with alternate page sizes? And how best can we embed
>that info into the sdbm so that later accesses to the file use the correct
>page size? My current thought is to embed a NULL-key record at the head
>of the file, with a specific known value element containing some identifier
>string such as "APR-SDBM" followed by two ints, the page and data len.
>If that record is missing (or created with default sizes), the file is treated
>as a standard sdbm with 1024/4096 sized pages.
? foo
? variable_sdbm.patch
Index: dbm/sdbm/sdbm.c
===================================================================
RCS file: /home/cvs/apr-util/dbm/sdbm/sdbm.c,v
retrieving revision 1.26
diff -u -r1.26 sdbm.c
--- dbm/sdbm/sdbm.c 1 Jan 2003 00:02:19 -0000 1.26
+++ dbm/sdbm/sdbm.c 16 Oct 2003 17:19:20 -0000
@@ -71,7 +71,6 @@
#include "sdbm_private.h"
#include <string.h> /* for memset() */
-#include <stdlib.h> /* for malloc() and free() */
/*
* forward
@@ -92,8 +91,8 @@
#define sdbm_dirfno(db) ((db)->dirf)
#define sdbm_pagfno(db) ((db)->pagf)
-#define OFF_PAG(off) (apr_off_t) (off) * PBLKSIZ
-#define OFF_DIR(off) (apr_off_t) (off) * DBLKSIZ
+#define OFF_PAG(db,off) ((apr_off_t) (off) * (db)->pblksiz)
+#define OFF_DIR(db,off) ((apr_off_t) (off) * (db)->dblksiz)
static long masks[] = {
000000000000, 000000000001, 000000000003, 000000000007,
@@ -120,7 +119,6 @@
(void) apr_file_unlock(db->dirf);
(void) apr_file_close(db->dirf);
(void) apr_file_close(db->pagf);
- free(db);
return APR_SUCCESS;
}
@@ -133,11 +131,28 @@
*pdb = NULL;
- db = malloc(sizeof(*db));
- memset(db, 0, sizeof(*db));
-
+ /* Allocate a single buffer for the structure and NULL it
+ */
+ db = apr_palloc(p, sizeof(apr_sdbm_t));
+ memset(db, 0, sizeof(apr_sdbm_t));
db->pool = p;
+ /* Determine the dblksiz/pblksiz (unimplemented),
+ * then allocate and null the buffers.
+ * Because we ultimately need to read these after the db is opened,
+ * they can't be allocated at the time we allocate db.
+ * The default sizes below indicate the "standard" sdbm parameters
+ * that are respected by perl and most sdbm implementations.
+ */
+ db->dblksiz = 4096;
+ db->pblksiz = 1024;
+
+ db->pagbuf = apr_palloc(db->pool, db->pblksiz * 3 + db->dblksiz);
+ memset(db->pagbuf, 0, db->pblksiz + db->dblksiz);
+ db->pagtmp = db->pagbuf + db->pblksiz;
+ db->pagwrk = db->pagtmp + db->pblksiz;
+ db->dirbuf = db->pagwrk + db->pblksiz;
+
/*
* adjust user flags so that WRONLY becomes RDWR,
* as required by this package. Also set our internal
@@ -205,7 +220,6 @@
if (db->pagf != NULL) {
(void) apr_file_close(db->pagf);
}
- free(db);
return status;
}
@@ -236,7 +250,7 @@
return status;
if ((status = getpage(db, exhash(key))) == APR_SUCCESS) {
- *val = getpair(db->pagbuf, key);
+ *val = getpair(db->pagbuf, key, db->pblksiz);
/* ### do we want a not-found result? */
}
@@ -248,10 +262,10 @@
static apr_status_t write_page(apr_sdbm_t *db, const char *buf, long pagno)
{
apr_status_t status;
- apr_off_t off = OFF_PAG(pagno);
+ apr_off_t off = OFF_PAG(db, pagno);
if ((status = apr_file_seek(db->pagf, APR_SET, &off)) == APR_SUCCESS)
- status = apr_file_write_full(db->pagf, buf, PBLKSIZ, NULL);
+ status = apr_file_write_full(db->pagf, buf, db->pblksiz, NULL);
return status;
}
@@ -270,7 +284,7 @@
return status;
if ((status = getpage(db, exhash(key))) == APR_SUCCESS) {
- if (!delpair(db->pagbuf, key))
+ if (!delpair(db->pagbuf, key, db->pblksiz))
/* ### should we define some APRUTIL codes? */
status = APR_EGENERAL;
else
@@ -295,9 +309,11 @@
return APR_EINVAL;
need = key.dsize + val.dsize;
/*
- * is the pair too big (or too small) for this database ??
+ * Is the pair too big (or too small) for this database ??
+ * This is derived from the original MAXPAIR calculation.
+ * At least MS compiler screamed that sizeof left the RHS unsigned.
*/
- if (need < 0 || need > PAIRMAX)
+ if ((need < 0) || (need > db->pblksiz - (int)sizeof(short) * 8))
return APR_EINVAL;
if ((status = apr_sdbm_lock(db, APR_FLOCK_EXCLUSIVE)) != APR_SUCCESS)
@@ -310,22 +326,23 @@
* first. If it is not there, ignore.
*/
if (flags == APR_SDBM_REPLACE)
- (void) delpair(db->pagbuf, key);
- else if (!(flags & APR_SDBM_INSERTDUP) && duppair(db->pagbuf, key)) {
+ (void) delpair(db->pagbuf, key, db->pblksiz);
+ else if (!(flags & APR_SDBM_INSERTDUP)
+ && duppair(db->pagbuf, key, db->pblksiz)) {
status = APR_EEXIST;
goto error;
}
/*
* if we do not have enough room, we have to split.
*/
- if (!fitpair(db->pagbuf, need))
+ if (!fitpair(db->pagbuf, need, db->pblksiz))
if ((status = makroom(db, hash, need)) != APR_SUCCESS)
goto error;
/*
* we have enough room or split is successful. insert the key,
* and update the page file.
*/
- (void) putpair(db->pagbuf, key, val);
+ (void) putpair(db->pagbuf, key, val, db->pblksiz);
status = write_page(db, db->pagbuf, db->pagbno);
}
@@ -344,8 +361,9 @@
static apr_status_t makroom(apr_sdbm_t *db, long hash, int need)
{
long newp;
- char twin[PBLKSIZ];
+ char *twin = db->pagtmp;
char *pag = db->pagbuf;
+ char *wrk = db->pagwrk;
char *new = twin;
register int smax = SPLTMAX;
apr_status_t status;
@@ -354,7 +372,7 @@
/*
* split the current page
*/
- (void) splpage(pag, new, db->hmask + 1);
+ (void) splpage(pag, new, wrk, db->hmask + 1, db->pblksiz);
/*
* address of the new page
*/
@@ -374,7 +392,7 @@
return status;
db->pagbno = newp;
- (void) memcpy(pag, new, PBLKSIZ);
+ (void) memcpy(pag, new, db->pblksiz);
}
else {
if ((status = write_page(db, new, newp)) != APR_SUCCESS)
@@ -386,7 +404,7 @@
/*
* see if we have enough room now
*/
- if (fitpair(pag, need))
+ if (fitpair(pag, need, db->pblksiz))
return APR_SUCCESS;
/*
* try again... update curbit and hmask as getpage would have
@@ -452,7 +470,7 @@
/*
* start at page 0
*/
- if ((status = read_from(db->pagf, db->pagbuf, OFF_PAG(0), PBLKSIZ))
+ if ((status = read_from(db->pagf, db->pagbuf, OFF_PAG(db, 0), db->pblksiz))
== APR_SUCCESS) {
db->pagbno = 0;
db->blkptr = 0;
@@ -512,11 +530,11 @@
* ### joe: this assumption was surely never correct? but
* ### we make it so in read_from anyway.
*/
- if ((status = read_from(db->pagf, db->pagbuf, OFF_PAG(pagb), PBLKSIZ))
+ if ((status = read_from(db->pagf, db->pagbuf, OFF_PAG(db, pagb),
db->pblksiz))
!= APR_SUCCESS)
return status;
- if (!chkpage(db->pagbuf))
+ if (!chkpage(db->pagbuf, db->pblksiz))
return APR_ENOSPC; /* ### better error? */
db->pagbno = pagb;
@@ -531,10 +549,10 @@
register long dirb;
c = dbit / BYTESIZ;
- dirb = c / DBLKSIZ;
+ dirb = c / db->dblksiz;
if (dirb != db->dirbno) {
- if (read_from(db->dirf, db->dirbuf, OFF_DIR(dirb), DBLKSIZ)
+ if (read_from(db->dirf, db->dirbuf, OFF_DIR(db, dirb), db->dblksiz)
!= APR_SUCCESS)
return 0;
@@ -543,7 +561,7 @@
debug(("dir read: %d\n", dirb));
}
- return db->dirbuf[c % DBLKSIZ] & (1 << dbit % BYTESIZ);
+ return db->dirbuf[c % db->dblksiz] & (1 << dbit % BYTESIZ);
}
static apr_status_t setdbit(apr_sdbm_t *db, long dbit)
@@ -554,10 +572,10 @@
apr_off_t off;
c = dbit / BYTESIZ;
- dirb = c / DBLKSIZ;
+ dirb = c / db->dblksiz;
if (dirb != db->dirbno) {
- if ((status = read_from(db->dirf, db->dirbuf, OFF_DIR(dirb), DBLKSIZ))
+ if ((status = read_from(db->dirf, db->dirbuf, OFF_DIR(db, dirb),
db->dblksiz))
!= APR_SUCCESS)
return status;
@@ -566,14 +584,14 @@
debug(("dir read: %d\n", dirb));
}
- db->dirbuf[c % DBLKSIZ] |= (1 << dbit % BYTESIZ);
+ db->dirbuf[c % db->dblksiz] |= (1 << dbit % BYTESIZ);
if (dbit >= db->maxbno)
- db->maxbno += DBLKSIZ * BYTESIZ;
+ db->maxbno += db->dblksiz * BYTESIZ;
- off = OFF_DIR(dirb);
+ off = OFF_DIR(db, dirb);
if ((status = apr_file_seek(db->dirf, APR_SET, &off)) == APR_SUCCESS)
- status = apr_file_write_full(db->dirf, db->dirbuf, DBLKSIZ, NULL);
+ status = apr_file_write_full(db->dirf, db->dirbuf, db->dblksiz, NULL);
return status;
}
@@ -587,7 +605,7 @@
apr_status_t status;
for (;;) {
db->keyptr++;
- *key = getnkey(db->pagbuf, db->keyptr);
+ *key = getnkey(db->pagbuf, db->keyptr, db->pblksiz);
if (key->dptr != NULL)
return APR_SUCCESS;
/*
@@ -597,7 +615,7 @@
*/
db->keyptr = 0;
if (db->pagbno != db->blkptr++) {
- apr_off_t off = OFF_PAG(db->blkptr);
+ apr_off_t off = OFF_PAG(db, db->blkptr);
if ((status = apr_file_seek(db->pagf, APR_SET, &off)
!= APR_SUCCESS))
return status;
@@ -605,10 +623,10 @@
db->pagbno = db->blkptr;
/* ### EOF acceptable here too? */
- if ((status = apr_file_read_full(db->pagf, db->pagbuf, PBLKSIZ, NULL))
+ if ((status = apr_file_read_full(db->pagf, db->pagbuf, db->pblksiz,
NULL))
!= APR_SUCCESS)
return status;
- if (!chkpage(db->pagbuf))
+ if (!chkpage(db->pagbuf, db->pblksiz))
return APR_EGENERAL; /* ### need better error */
}
@@ -623,4 +641,3 @@
*/
return (db->flags & SDBM_RDONLY) != 0;
}
-
Index: dbm/sdbm/sdbm_pair.c
===================================================================
RCS file: /home/cvs/apr-util/dbm/sdbm/sdbm_pair.c,v
retrieving revision 1.6
diff -u -r1.6 sdbm_pair.c
--- dbm/sdbm/sdbm_pair.c 1 Jan 2003 00:02:19 -0000 1.6
+++ dbm/sdbm/sdbm_pair.c 16 Oct 2003 17:19:20 -0000
@@ -75,7 +75,7 @@
/*
* forward
*/
-static int seepair(char *, int, char *, int);
+static int seepair(char *, int, char *, int, int);
/*
* page format:
@@ -98,16 +98,17 @@
*/
int
-fitpair(pag, need)
+fitpair(pag, need, pblksz)
char *pag;
int need;
+int pblksz;
{
- register int n;
- register int off;
- register int avail;
register short *ino = (short *) pag;
+ register int n = ino[0];
+ register int avail;
+ register int off;
- off = ((n = ino[0]) > 0) ? ino[n] : PBLKSIZ;
+ off = (n > 0) ? ino[n] : pblksz;
avail = off - (n + 1) * sizeof(short);
need += 2 * sizeof(short);
@@ -117,16 +118,17 @@
}
void
-putpair(pag, key, val)
+putpair(pag, key, val, pblksz)
char *pag;
apr_sdbm_datum_t key;
apr_sdbm_datum_t val;
+int pblksz;
{
- register int n;
- register int off;
register short *ino = (short *) pag;
+ register int n = ino[0];
+ register int off;
- off = ((n = ino[0]) > 0) ? ino[n] : PBLKSIZ;
+ off = (n > 0) ? ino[n] : pblksz;
/*
* enter the key first
*/
@@ -146,19 +148,20 @@
}
apr_sdbm_datum_t
-getpair(pag, key)
+getpair(pag, key, pblksz)
char *pag;
apr_sdbm_datum_t key;
+int pblksz;
{
- register int i;
- register int n;
apr_sdbm_datum_t val;
register short *ino = (short *) pag;
+ register int n = ino[0];
+ register int i;
- if ((n = ino[0]) == 0)
+ if (n == 0)
return sdbm_nullitem;
- if ((i = seepair(pag, n, key.dptr, key.dsize)) == 0)
+ if ((i = seepair(pag, n, key.dptr, key.dsize, pblksz)) == 0)
return sdbm_nullitem;
val.dptr = pag + ino[i + 1];
@@ -167,28 +170,31 @@
}
int
-duppair(pag, key)
+duppair(pag, key, pblksz)
char *pag;
apr_sdbm_datum_t key;
+int pblksz;
{
register short *ino = (short *) pag;
- return ino[0] > 0 && seepair(pag, ino[0], key.dptr, key.dsize) > 0;
+ return ino[0] > 0
+ && seepair(pag, ino[0], key.dptr, key.dsize, pblksz) > 0;
}
apr_sdbm_datum_t
-getnkey(pag, num)
+getnkey(pag, num, pblksz)
char *pag;
int num;
+int pblksz;
{
+ register short *ino = (short *) pag;
apr_sdbm_datum_t key;
register int off;
- register short *ino = (short *) pag;
num = num * 2 - 1;
if (ino[0] == 0 || num > ino[0])
return sdbm_nullitem;
- off = (num > 1) ? ino[num - 1] : PBLKSIZ;
+ off = (num > 1) ? ino[num - 1] : pblksz;
key.dptr = pag + ino[num];
key.dsize = off - ino[num];
@@ -197,18 +203,19 @@
}
int
-delpair(pag, key)
+delpair(pag, key, pblksz)
char *pag;
apr_sdbm_datum_t key;
+int pblksz;
{
- register int n;
- register int i;
register short *ino = (short *) pag;
+ register int n = ino[0];
+ register int i;
- if ((n = ino[0]) == 0)
+ if (n == 0)
return 0;
- if ((i = seepair(pag, n, key.dptr, key.dsize)) == 0)
+ if ((i = seepair(pag, n, key.dptr, key.dsize, pblksz)) == 0)
return 0;
/*
* found the key. if it is the last entry
@@ -219,7 +226,7 @@
*/
if (i < n - 1) {
register int m;
- register char *dst = pag + (i == 1 ? PBLKSIZ : ino[i - 1]);
+ register char *dst = pag + (i == 1 ? pblksz : ino[i - 1]);
register char *src = pag + ino[i + 1];
register int zoo = dst - src;
@@ -269,14 +276,15 @@
* return 0 if not found.
*/
static int
-seepair(pag, n, key, siz)
+seepair(pag, n, key, siz, pblksz)
char *pag;
register int n;
register char *key;
register int siz;
+int pblksz;
{
register int i;
- register int off = PBLKSIZ;
+ register int off = pblksz;
register short *ino = (short *) pag;
for (i = 1; i < n; i += 2) {
@@ -289,22 +297,24 @@
}
void
-splpage(pag, new, sbit)
+splpage(pag, new, wrk, sbit, pblksz)
char *pag;
char *new;
+char *wrk;
long sbit;
+int pblksz;
{
apr_sdbm_datum_t key;
apr_sdbm_datum_t val;
register int n;
- register int off = PBLKSIZ;
- char cur[PBLKSIZ];
+ register int off = pblksz;
+ char *cur = wrk;
register short *ino = (short *) cur;
- (void) memcpy(cur, pag, PBLKSIZ);
- (void) memset(pag, 0, PBLKSIZ);
- (void) memset(new, 0, PBLKSIZ);
+ (void) memcpy(cur, pag, pblksz);
+ (void) memset(pag, 0, pblksz);
+ (void) memset(new, 0, pblksz);
n = ino[0];
for (ino++; n > 0; ino += 2) {
@@ -315,7 +325,8 @@
/*
* select the page pointer (by looking at sbit) and insert
*/
- (void) putpair((exhash(key) & sbit) ? new : pag, key, val);
+ (void) putpair((exhash(key) & sbit) ? new : pag,
+ key, val, pblksz);
off = ino[1];
n -= 2;
@@ -333,18 +344,19 @@
* this could be made more rigorous.
*/
int
-chkpage(pag)
+chkpage(pag, pblksz)
char *pag;
+int pblksz;
{
- register int n;
- register int off;
register short *ino = (short *) pag;
+ register int n = ino[0];
+ register int off;
- if ((n = ino[0]) < 0 || n > PBLKSIZ / sizeof(short))
+ if (n < 0 || n > pblksz / (int)sizeof(short))
return 0;
if (n > 0) {
- off = PBLKSIZ;
+ off = pblksz;
for (ino++; n > 0; ino += 2) {
if (ino[0] > off || ino[1] > off ||
ino[1] > ino[0])
Index: dbm/sdbm/sdbm_pair.h
===================================================================
RCS file: /home/cvs/apr-util/dbm/sdbm/sdbm_pair.h,v
retrieving revision 1.5
diff -u -r1.5 sdbm_pair.h
--- dbm/sdbm/sdbm_pair.h 1 Jan 2003 00:02:19 -0000 1.5
+++ dbm/sdbm/sdbm_pair.h 16 Oct 2003 17:19:20 -0000
@@ -65,14 +65,14 @@
#define putpair sdbm__putpair
#define splpage sdbm__splpage
-int fitpair(char *, int);
-void putpair(char *, apr_sdbm_datum_t, apr_sdbm_datum_t);
-apr_sdbm_datum_t getpair(char *, apr_sdbm_datum_t);
-int delpair(char *, apr_sdbm_datum_t);
-int chkpage (char *);
-apr_sdbm_datum_t getnkey(char *, int);
-void splpage(char *, char *, long);
-int duppair(char *, apr_sdbm_datum_t);
+int fitpair(char *, int, int);
+void putpair(char *, apr_sdbm_datum_t, apr_sdbm_datum_t, int);
+apr_sdbm_datum_t getpair(char *, apr_sdbm_datum_t, int);
+int delpair(char *, apr_sdbm_datum_t, int);
+int chkpage (char *, int);
+apr_sdbm_datum_t getnkey(char *, int, int);
+void splpage(char *, char *, char *, long, int);
+int duppair(char *, apr_sdbm_datum_t, int);
#endif /* SDBM_PAIR_H */
Index: dbm/sdbm/sdbm_private.h
===================================================================
RCS file: /home/cvs/apr-util/dbm/sdbm/sdbm_private.h,v
retrieving revision 1.10
diff -u -r1.10 sdbm_private.h
--- dbm/sdbm/sdbm_private.h 1 Jan 2003 00:02:19 -0000 1.10
+++ dbm/sdbm/sdbm_private.h 16 Oct 2003 17:19:20 -0000
@@ -66,17 +66,7 @@
#include "apr_file_io.h"
#include "apr_errno.h" /* for apr_status_t */
-#if 0
-/* if the block/page size is increased, it breaks perl apr_sdbm_t
compatibility */
-#define DBLKSIZ 16384
-#define PBLKSIZ 8192
-#define PAIRMAX 8008 /* arbitrary on PBLKSIZ-N */
-#else
-#define DBLKSIZ 4096
-#define PBLKSIZ 1024
-#define PAIRMAX 1008 /* arbitrary on PBLKSIZ-N */
-#endif
-#define SPLTMAX 10 /* maximum allowed splits */
+#define SPLTMAX 10 /* maximum allowed splits */
/* for apr_sdbm_t.flags */
#define SDBM_RDONLY 0x1 /* data base open read-only */
@@ -96,9 +86,13 @@
int keyptr; /* current key for nextkey */
long blkno; /* current page to read/write */
long pagbno; /* current page in pagbuf */
- char pagbuf[PBLKSIZ]; /* page file block buffer */
+ int pblksiz; /* sdbm's data blocksize (d:1024) */
+ char *pagbuf; /* page file block buffer */
+ char *pagtmp; /* page file temp workspace */
+ char *pagwrk; /* page file split workspace */
long dirbno; /* current block in dirbuf */
- char dirbuf[DBLKSIZ]; /* directory file block buffer */
+ int dblksiz; /* sdbm's dir blocksize (d:4096) */
+ char *dirbuf; /* directory file block buffer */
int lckcnt; /* number of calls to sdbm_lock */
};
