*** ./contrib/dict_preload/dict_preload.c.orig 2010-03-18 17:00:33.000000000 +0100 --- ./contrib/dict_preload/dict_preload.c 2010-04-01 14:40:58.980976853 +0200 *************** *** 0 **** --- 1,181 ---- + /*------------------------------------------------------------------------- + * + * dict_preload.c + * preloaded dictionary + * + * Copyright (c) 2007-2010, PostgreSQL Global Development Group + * + * IDENTIFICATION + * $PostgreSQL: pgsql/contrib/dict_preload/dict_preload.c,v 1.6 2010/01/02 16:57:32 momjian Exp $ + * + *------------------------------------------------------------------------- + */ + #include + + #include "postgres.h" + + #include "commands/defrem.h" + #include "fmgr.h" + #include "miscadmin.h" + #include "nodes/makefuncs.h" + #include "nodes/value.h" + #include "tsearch/ts_public.h" + #include "tsearch/ts_utils.h" + #include "tsearch/dicts/spell.h" + #include "utils/guc.h" + #include "utils/memutils.h" + + PG_MODULE_MAGIC; + + char *preload_dictfile = NULL; + char *preload_afffile = NULL; + char *preload_stopwords = NULL; + + typedef struct + { + StopList stoplist; + IspellDict obj; + } DictISpell; + + MemoryContext preload_ctx = NULL; + + DictISpell *preload_dict = NULL; + + PG_FUNCTION_INFO_V1(dpreloaddict_init); + Datum dpreloaddict_init(PG_FUNCTION_ARGS); + + PG_FUNCTION_INFO_V1(dpreloaddict_lexize); + Datum dpreloaddict_lexize(PG_FUNCTION_ARGS); + + void _PG_init(void); + void _PG_fini(void); + + static DictISpell * + load_dictionary(void) + { + List *dictopt = NIL; + FunctionCallInfoData fcinfo; + DictISpell *result; + + /* + * read parameters for preloaded dictionary + */ + if (preload_dictfile != NULL) + dictopt = lappend(dictopt, makeDefElem("DictFile", + (Node *) makeString(preload_dictfile))); + if (preload_afffile != NULL) + dictopt = lappend(dictopt, makeDefElem("AffFile", + (Node *) makeString(preload_afffile))); + if (preload_stopwords != NULL) + dictopt = lappend(dictopt, makeDefElem("StopWords", + (Node *) makeString(preload_stopwords))); + /* + * Initialise ispell dictionary + */ + InitFunctionCallInfoData(fcinfo, NULL, 1, NULL, NULL); + fcinfo.arg[0] = PointerGetDatum(dictopt); + fcinfo.argnull[0] = false; + + result = (DictISpell *) DatumGetPointer(dispell_init(&fcinfo)); + + + MemoryContextStats(CurrentMemoryContext); + + return result; + } + + Datum + dpreloaddict_init(PG_FUNCTION_ARGS) + { + static bool firsttime = true; + + /* + * dpreloaddict_init can be called more times: + * CREATE TEXT SEARCH DICTIONARY + * DROP TEXT SEARCH DICTIONARY + * CREATE TEXT SEARCH DICTIONARY + * ... + */ + if (firsttime) + { + /* In this moment, dictionary have to be loaded */ + Assert(MemoryContextIsValid(preload_ctx)); + Assert(preload_dict != NULL); + + preload_ctx = NULL; + firsttime = false; + + return PointerGetDatum(preload_dict); + } + else + return PointerGetDatum(load_dictionary()); + } + + Datum + dpreloaddict_lexize(PG_FUNCTION_ARGS) + { + return dispell_lexize(fcinfo); + } + + /* + * Module load callback + */ + void + _PG_init() + { + MemoryContext oldctx; + static bool inited = false; + GucContext guc_ctx; + + if (inited) + return; + else + inited = true; + + guc_ctx = process_shared_preload_libraries_in_progress ? + PGC_POSTMASTER : PGC_SUSET; + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.dictfile", + "name of file of preloaded ispell dictionary", + NULL, + &preload_dictfile, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.afffile", + "name of file of preloaded ispell affix", + NULL, + &preload_afffile, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* Define custom GUC variables. */ + DefineCustomStringVariable("dict_preload.stopwords", + "name of file of preloaded ispell stopwords", + NULL, + &preload_stopwords, + NULL, + guc_ctx, 0, + NULL, NULL); + + /* preload dictionary */ + Assert(preload_ctx == NULL); + Assert(preload_dict == NULL); + + preload_ctx = MMapAllocSetContextCreate(NULL, "Ispell dictionary preload context", + 512 * 1024, + 1024 * 1024 *4, + 1024 * 1024 *8, + PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANON, + -1); + oldctx = MemoryContextSwitchTo(preload_ctx); + + preload_dict = load_dictionary(); + + MemoryContextSwitchTo(oldctx); + } *** ./contrib/dict_preload/dict_preload.sql.in.orig 2010-03-18 17:00:52.000000000 +0100 --- ./contrib/dict_preload/dict_preload.sql.in 2010-03-19 08:24:33.000000000 +0100 *************** *** 0 **** --- 1,19 ---- + /* $PostgreSQL: pgsql/contrib/dict_int/dict_int.sql.in,v 1.3 2007/11/13 04:24:27 momjian Exp $ */ + + -- Adjust this setting to control where the objects get created. + SET search_path = public; + + CREATE OR REPLACE FUNCTION dpreloaddict_init(internal) + RETURNS internal + AS 'MODULE_PATHNAME' + LANGUAGE C STRICT; + + CREATE OR REPLACE FUNCTION dpreloaddict_lexize(internal, internal, internal, internal) + RETURNS internal + AS 'MODULE_PATHNAME' + LANGUAGE C STRICT; + + CREATE TEXT SEARCH TEMPLATE preloaddict( + LEXIZE = dpreloaddict_lexize, + INIT = dpreloaddict_init + ); *** ./contrib/dict_preload/uninstall_dict_preload.sql.orig 2010-03-18 17:00:58.000000000 +0100 --- ./contrib/dict_preload/uninstall_dict_preload.sql 2010-03-18 13:52:49.000000000 +0100 *************** *** 0 **** --- 1,10 ---- + /* $PostgreSQL: pgsql/contrib/dict_int/uninstall_dict_int.sql,v 1.3 2007/11/13 04:24:27 momjian Exp $ */ + + -- Adjust this setting to control where the objects get dropped. + SET search_path = public; + + DROP TEXT SEARCH TEMPLATE preloaddict_template CASCADE; + + DROP FUNCTION dpreloaddict_init(internal); + + DROP FUNCTION dpreloaddict_lexize(internal,internal,internal,internal); *** ./src/backend/tsearch/spell.c.orig 2010-01-02 17:57:53.000000000 +0100 --- ./src/backend/tsearch/spell.c 2010-04-01 15:09:06.512848296 +0200 *************** *** 31,36 **** --- 31,41 ---- #define tmpalloc(sz) MemoryContextAlloc(tmpCtx, (sz)) #define tmpalloc0(sz) MemoryContextAllocZero(tmpCtx, (sz)) + static void *simple_alloc_ptr; + static Size simple_alloc_free; + + #define SIMPLE_ALLOC_BLOCKSIZE (1024 * 1024 * 2) + static void checkTmpCtx(void) { *************** *** 45,50 **** --- 50,56 ---- ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); + simple_alloc_free = 0; } else tmpCtx = CurrentMemoryContext->firstchild; *************** *** 63,68 **** --- 69,118 ---- return dst; } + static void * + simple_alloc(Size size) + { + void *ptr; + + size = MAXALIGN(size); + + if (size > simple_alloc_free) + { + simple_alloc_ptr = palloc(SIMPLE_ALLOC_BLOCKSIZE); + simple_alloc_free = SIMPLE_ALLOC_BLOCKSIZE; + } + + ptr = simple_alloc_ptr; + simple_alloc_ptr = (char *) simple_alloc_ptr + size; + simple_alloc_free -= size; + + return ptr; + } + + static void * + simple_alloc0(Size size) + { + void *ptr; + + ptr = simple_alloc(size); + memset(ptr, 0, size); + + return ptr; + } + + static char * + simple_strdup(char *str) + { + char *result; + int len; + + len = strlen(str); + result = simple_alloc(len + 1); + memcpy(result, str, len + 1); + + return result; + } + #define MAX_NORM 1024 #define MAXNORMLEN 256 *************** *** 375,383 **** Affix->flag = flag; Affix->type = type; ! Affix->find = (find && *find) ? pstrdup(find) : VoidString; if ((Affix->replen = strlen(repl)) > 0) ! Affix->repl = pstrdup(repl); else Affix->repl = VoidString; Conf->naffixes++; --- 425,433 ---- Affix->flag = flag; Affix->type = type; ! Affix->find = (find && *find) ? simple_strdup(find) : VoidString; if ((Affix->replen = strlen(repl)) > 0) ! Affix->repl = simple_strdup(repl); else Affix->repl = VoidString; Conf->naffixes++; *************** *** 833,839 **** } ptr = Conf->AffixData + Conf->nAffixData; ! *ptr = palloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) + 1 /* space */ + 1 /* \0 */ ); sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]); ptr++; --- 883,889 ---- } ptr = Conf->AffixData + Conf->nAffixData; ! *ptr = simple_alloc(strlen(Conf->AffixData[a1]) + strlen(Conf->AffixData[a2]) + 1 /* space */ + 1 /* \0 */ ); sprintf(*ptr, "%s %s", Conf->AffixData[a1], Conf->AffixData[a2]); ptr++; *************** *** 878,884 **** if (!nchar) return NULL; ! rs = (SPNode *) palloc0(SPNHDRSZ + nchar * sizeof(SPNodeData)); rs->length = nchar; data = rs->data; --- 928,934 ---- if (!nchar) return NULL; ! rs = (SPNode *) simple_alloc0(SPNHDRSZ + nchar * sizeof(SPNodeData)); rs->length = nchar; data = rs->data; *************** *** 974,980 **** { curaffix++; Assert(curaffix < naffix); ! Conf->AffixData[curaffix] = pstrdup(Conf->Spell[i]->p.flag); } Conf->Spell[i]->p.d.affix = curaffix; --- 1024,1030 ---- { curaffix++; Assert(curaffix < naffix); ! Conf->AffixData[curaffix] = simple_strdup(Conf->Spell[i]->p.flag); } Conf->Spell[i]->p.d.affix = curaffix; *************** *** 1014,1020 **** aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1)); naff = 0; ! rs = (AffixNode *) palloc0(ANHRDSZ + nchar * sizeof(AffixNodeData)); rs->length = nchar; data = rs->data; --- 1064,1070 ---- aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1)); naff = 0; ! rs = (AffixNode *) simple_alloc0(ANHRDSZ + nchar * sizeof(AffixNodeData)); rs->length = nchar; data = rs->data; *************** *** 1030,1036 **** if (naff) { data->naff = naff; ! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } --- 1080,1086 ---- if (naff) { data->naff = naff; ! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } *************** *** 1050,1056 **** if (naff) { data->naff = naff; ! data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } --- 1100,1106 ---- if (naff) { data->naff = naff; ! data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * naff); memcpy(data->aff, aff, sizeof(AFFIX *) * naff); naff = 0; } *************** *** 1067,1073 **** cnt = 0; int start = (issuffix) ? startsuffix : 0; int end = (issuffix) ? Conf->naffixes : startsuffix; ! AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData)); Affix->length = 1; Affix->isvoid = 1; --- 1117,1123 ---- cnt = 0; int start = (issuffix) ? startsuffix : 0; int end = (issuffix) ? Conf->naffixes : startsuffix; ! AffixNode *Affix = (AffixNode *) simple_alloc0(ANHRDSZ + sizeof(AffixNodeData)); Affix->length = 1; Affix->isvoid = 1; *************** *** 1091,1097 **** if (cnt == 0) return; ! Affix->data->aff = (AFFIX **) palloc(sizeof(AFFIX *) * cnt); Affix->data->naff = (uint32) cnt; cnt = 0; --- 1141,1147 ---- if (cnt == 0) return; ! Affix->data->aff = (AFFIX **) simple_alloc(sizeof(AFFIX *) * cnt); Affix->data->naff = (uint32) cnt; cnt = 0; *** ./src/backend/utils/mmgr/aset.c.orig 2010-02-26 03:01:14.000000000 +0100 --- ./src/backend/utils/mmgr/aset.c 2010-04-01 14:16:33.719848482 +0200 *************** *** 62,67 **** --- 62,71 ---- *------------------------------------------------------------------------- */ + #include + #include + #include + #include "postgres.h" #include "utils/memutils.h" *************** *** 122,127 **** --- 126,155 ---- typedef void *AllocPointer; /* + * This is the virtual table for external memory methods + */ + typedef struct MemoryAccessMethods + { + void *data; /* pointer on data used for external allocator */ + void *(*alloc) (Size size, void *data); + void *(*realloc) (void *ptr, Size size, void *data); + void (*free) (void *ptr, void *data); + } MemoryAccessMethods; + + typedef struct MMapMemoryContextData + { + int prot; + int flags; + int fd; + } MMapMemoryContextData; + + #define ALLOC(_ctx, size) (*_ctx->methods->alloc) (size, _ctx->methods->data) + #define REALLOC(_ctx, ptr, size) (*_ctx->methods->realloc) (ptr, size, _ctx->methods->data) + #define FREE(_ctx, ptr) (*_ctx->methods->free) (ptr, _ctx->methods->data) + + #define PAGE_ALIGN(size) (((size) / getpagesize() + 1) * getpagesize()) + + /* * AllocSetContext is our standard implementation of MemoryContext. * * Note: isReset means there is nothing for AllocSetReset to do. This is *************** *** 143,148 **** --- 171,177 ---- Size nextBlockSize; /* next block size to allocate */ Size allocChunkLimit; /* effective chunk size limit */ AllocBlock keeper; /* if not NULL, keep this block over resets */ + MemoryAccessMethods *methods; } AllocSetContext; typedef AllocSetContext *AllocSet; *************** *** 220,225 **** --- 249,270 ---- static void AllocSetCheck(MemoryContext context); #endif + static void *memory_alloc(Size size, void *data); + static void *memory_realloc(void *ptr, Size size, void *data); + static void memory_free(void *ptr, void *data); + + static void *mmap_alloc(Size size, void *data); + static void *mmap_realloc(void *ptr, Size size, void *data); + static void mmap_free(void *ptr, void *data); + + static MemoryContext + _AllocSetContextCreate(MemoryContext parent, const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize, + MemoryAccessMethods *methods); + + /* * This is the virtual function table for AllocSet contexts. */ *************** *** 238,243 **** --- 283,295 ---- #endif }; + static MemoryAccessMethods MemoryAccess = { + NULL, /* there are no special data */ + memory_alloc, + memory_realloc, + memory_free + }; + /* * Table for AllocSetFreeIndex */ *************** *** 266,271 **** --- 318,324 ---- #define AllocAllocInfo(_cxt, _chunk) #endif + /* ---------- * AllocSetFreeIndex - * *************** *** 353,358 **** --- 406,427 ---- Size initBlockSize, Size maxBlockSize) { + return _AllocSetContextCreate(parent, name, + minContextSize, + initBlockSize, + maxBlockSize, + &MemoryAccess); + } + + + static MemoryContext + _AllocSetContextCreate(MemoryContext parent, + const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize, + MemoryAccessMethods *methods) + { AllocSet context; /* Do the type-independent part of context creation */ *************** *** 376,381 **** --- 445,451 ---- context->initBlockSize = initBlockSize; context->maxBlockSize = maxBlockSize; context->nextBlockSize = initBlockSize; + context->methods = methods; /* * Compute the allocation chunk size limit for this context. It can't be *************** *** 399,405 **** Size blksize = MAXALIGN(minContextSize); AllocBlock block; ! block = (AllocBlock) malloc(blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); --- 469,475 ---- Size blksize = MAXALIGN(minContextSize); AllocBlock block; ! block = (AllocBlock) ALLOC(context, blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); *************** *** 503,509 **** /* Wipe freed memory for debugging purposes */ memset(block, 0x7F, block->freeptr - ((char *) block)); #endif ! free(block); } block = next; } --- 573,579 ---- /* Wipe freed memory for debugging purposes */ memset(block, 0x7F, block->freeptr - ((char *) block)); #endif ! FREE(set, block); } block = next; } *************** *** 578,584 **** { chunk_size = MAXALIGN(size); blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; ! block = (AllocBlock) malloc(blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); --- 648,654 ---- { chunk_size = MAXALIGN(size); blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; ! block = (AllocBlock) ALLOC(set, blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); *************** *** 748,754 **** blksize <<= 1; /* Try to allocate it */ ! block = (AllocBlock) malloc(blksize); /* * We could be asking for pretty big blocks here, so cope if malloc --- 818,824 ---- blksize <<= 1; /* Try to allocate it */ ! block = (AllocBlock) ALLOC(set, blksize); /* * We could be asking for pretty big blocks here, so cope if malloc *************** *** 759,765 **** blksize >>= 1; if (blksize < required_size) break; ! block = (AllocBlock) malloc(blksize); } if (block == NULL) --- 829,835 ---- blksize >>= 1; if (blksize < required_size) break; ! block = (AllocBlock) ALLOC(set, blksize); } if (block == NULL) *************** *** 870,876 **** /* Wipe freed memory for debugging purposes */ memset(block, 0x7F, block->freeptr - ((char *) block)); #endif ! free(block); } else { --- 940,946 ---- /* Wipe freed memory for debugging purposes */ memset(block, 0x7F, block->freeptr - ((char *) block)); #endif ! FREE(set, block); } else { *************** *** 967,973 **** /* Do the realloc */ chksize = MAXALIGN(size); blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; ! block = (AllocBlock) realloc(block, blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); --- 1037,1043 ---- /* Do the realloc */ chksize = MAXALIGN(size); blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; ! block = (AllocBlock) REALLOC(set, block, blksize); if (block == NULL) { MemoryContextStats(TopMemoryContext); *************** *** 1199,1201 **** --- 1269,1418 ---- } #endif /* MEMORY_CONTEXT_CHECKING */ + + /* + * Methods for access to glibc space + */ + static void * + memory_alloc(Size size, void *data) + { + return malloc(size); + } + + static void * + memory_realloc(void *ptr, Size size, void *data) + { + return realloc(ptr, size); + } + + static void + memory_free(void *ptr, void *data) + { + free(ptr); + } + + + /* + * Methods for access to mmap space + */ + MemoryContext + MMapAllocSetContextCreate(MemoryContext parent, + const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize, + int prot, + int flags, + int fd) + { + MMapMemoryContextData *mmap_data; + MemoryAccessMethods *mmap_access; + + mmap_data = MemoryContextAlloc(TopMemoryContext, + sizeof(MMapMemoryContextData)); + mmap_data->prot = prot; + mmap_data->flags = flags; + mmap_data->fd = fd; + + mmap_access = MemoryContextAlloc(TopMemoryContext, + sizeof(MemoryAccessMethods)); + + mmap_access->data = mmap_data; + mmap_access->alloc = mmap_alloc; + mmap_access->realloc = mmap_realloc; + mmap_access->free = mmap_free; + + return _AllocSetContextCreate(parent, name, + minContextSize, + initBlockSize, + maxBlockSize, + mmap_access); + } + + static void * + mmap_alloc(Size size, void *data) + { + void *ptr; + MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data; + + Assert(mmdata != NULL); + + size = PAGE_ALIGN(MAXALIGN(size) + MAXALIGN(sizeof(size))); + + ptr = mmap(NULL, size, mmdata->prot, mmdata->flags, + -1, 0); + + if (ptr == MAP_FAILED) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) size))); + + *((Size*) ptr) = size; + return (char *) ptr + MAXALIGN(sizeof(Size)); + } + + static void * + mmap_realloc(void *ptr, Size size, void *data) + { + MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data; + Size oldsize; + void *result = ptr; + + ptr = (char *) ptr - MAXALIGN(sizeof(Size)); + + oldsize = *((Size *) ptr); + size = PAGE_ALIGN(MAXALIGN(size) + MAXALIGN(sizeof(Size))); + + Assert(mmdata != NULL); + + /* + * Attention - on linux you cannot call mremap together with + * shared + * + * ptr = mremap(ptr, oldsize, + * size, + * MREMAP_MAYMOVE); + */ + if (size > oldsize) + { + void *newptr; + Size newsize = oldsize; + + while (newsize < size) + newsize = PAGE_ALIGN(2 * newsize); + + newptr = mmap(NULL, newsize, + mmdata->prot, mmdata->flags, mmdata->fd, + 0); + + if (newptr == MAP_FAILED) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %lu.", + (unsigned long) newsize))); + memcpy(newptr, ptr, oldsize); + *((Size*) newptr) = newsize; + + munmap(ptr, oldsize); + result = (char *) newptr + MAXALIGN(sizeof(Size)); + } + + return result; + } + + static void + mmap_free(void *ptr, void *data) + { + Size size; + MMapMemoryContextData *mmdata = (MMapMemoryContextData *) data; + + Assert(mmdata != NULL); + + ptr = (char *) ptr - MAXALIGN(sizeof(Size)); + size = *((Size*) ptr); + + munmap(ptr, size); + } *** ./src/include/utils/memutils.h.orig 2010-01-02 17:58:10.000000000 +0100 --- ./src/include/utils/memutils.h 2010-04-01 14:18:33.319326916 +0200 *************** *** 120,125 **** --- 120,134 ---- Size initBlockSize, Size maxBlockSize); + extern MemoryContext MMapAllocSetContextCreate(MemoryContext parent, + const char *name, + Size minContextSize, + Size initBlockSize, + Size maxBlockSize, + int prot, + int flags, + int fd); + /* * Recommended default alloc parameters, suitable for "ordinary" contexts * that might hold quite a lot of data.