2011/6/17 Mark Kirkwood <mark.kirkw...@catalyst.net.nz>:
> On 17/06/11 13:08, Mark Kirkwood wrote:
>>
>> On 17/06/11 09:49, Cédric Villemain wrote:
>>>
>>> I have issues applying it.
>>> Please can you remove trailing space?
>>> Also, you can generate a cool patch like this :
>>>
>>> get git-external-diff from postgres/src/tools to /usr/lib/git-core/
>>> chmod +x it
>>> export GIT_EXTERNAL_DIFF=git-external-diff
>>> git format-patch --ext-diff origin
>
> I think I have the trailing spaces removed, and patch is updated for the
> variable renaming recently done in fd.c
>
> I have no idea why I can't get the git apply to work (obviously I have
> exceeded by git foo by quite a ways), but it should apply for you I hope (as
> it patches fine).
>
If I didn't made mistake the attached patch does not have trailling
space anymore and I did a minor cosmetic in FileClose. It is not in
the expected format required by postgresql commiters but can be
applyed with git apply...
It looks like the issue is that patch generated with the git-ext-diff
can not be git applyed (they need to use patch).
Either I did something wrong or git-ext-diff format is not so great.
I didn't test and all yet. From reading, the patch looks sane. I'll
review it later this day or this week-end.
--
Cédric Villemain 2ndQuadrant
http://2ndQuadrant.fr/ PostgreSQL : Expertise, Formation et Support
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e835e4b..80d7c35 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1025,6 +1025,43 @@ SET ENABLE_SEQSCAN TO OFF;
</variablelist>
</sect2>
+ <sect2 id="runtime-config-resource-disk">
+ <title>Disk</title>
+ <variablelist>
+
+ <varlistentry id="guc-temp-file-limit" xreflabel="temp_file_limit">
+ <term><varname>temp_file_limit</varname> (<type>integer</type>)</term>
+ <indexterm>
+ <primary><varname>temp_file_limit</> configuration parameter</primary>
+ </indexterm>
+ <listitem>
+ <para>
+ Specifies the amount of disk space used by internal sort operations
+ and hash tables whist writing to temporary disk files. The value
+ defaults to unlimited (<literal>-1</>). Values larger than zero
+ constraint the temporary file space usage to be that number of
+ kilobytes.
+ </para>
+ <para>
+ A given sort or hash operation may write a number of temporary files,
+ the total space used by all the files produced by one backend is
+ constrained to be this value or less. If further bytes are written
+ the current query is canceled. Only superusers can change this
+ setting.
+ </para>
+ <para>
+ It should be noted that this parameter does <emphasis>not</emphasis>
+ constrain disk space used for temporary table storage. However if
+ the temporary table is created from a query then the any sort
+ and hash files used in query execution will have their space
+ controlled as above.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ </variablelist>
+ </sect2>
+
<sect2 id="runtime-config-resource-kernel">
<title>Kernel Resource Usage</title>
<variablelist>
diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c
index 820e6db..5c00889 100644
--- a/src/backend/storage/file/fd.c
+++ b/src/backend/storage/file/fd.c
@@ -131,6 +131,11 @@ static int max_safe_fds = 32; /* default if not changed */
/* Flag to tell whether there are files to close/delete at end of transaction */
static bool have_pending_fd_cleanup = false;
+/*
+ * Track the total size of all temporary files
+ */
+static double temporary_files_size = 0.0;
+
typedef struct vfd
{
int fd; /* current FD, or VFD_CLOSED if none */
@@ -140,6 +145,7 @@ typedef struct vfd
File lruMoreRecently; /* doubly linked recency-of-use list */
File lruLessRecently;
off_t seekPos; /* current logical file position */
+ off_t fileSize; /* current size of file */
char *fileName; /* name of file, or NULL for unused VFD */
/* NB: fileName is malloc'd, and must be free'd when closing the VFD */
int fileFlags; /* open(2) flags for (re)opening the file */
@@ -887,6 +893,7 @@ PathNameOpenFile(FileName fileName, int fileFlags, int fileMode)
vfdP->fileFlags = fileFlags & ~(O_CREAT | O_TRUNC | O_EXCL);
vfdP->fileMode = fileMode;
vfdP->seekPos = 0;
+ vfdP->fileSize = 0;
vfdP->fdstate = 0x0;
vfdP->resowner = NULL;
@@ -1123,6 +1130,13 @@ FileClose(File file)
if (unlink(vfdP->fileName))
elog(LOG, "could not unlink file \"%s\": %m", vfdP->fileName);
}
+
+ if (temp_file_limit >= 0)
+ {
+ /* subtract the unlinked file size from the total */
+ temporary_files_size -= (double)vfdP->fileSize;
+ vfdP->fileSize = 0;
+ }
}
/* Unregister it from the resource owner */
@@ -1251,7 +1265,27 @@ retry:
errno = ENOSPC;
if (returnCode >= 0)
+ {
VfdCache[file].seekPos += returnCode;
+
+ if (temp_file_limit >= 0 && VfdCache[file].fdstate & FD_TEMPORARY)
+ {
+ /*
+ * if we increase the file size, add it to the total, then check it is
+ * not too big
+ */
+ if (VfdCache[file].seekPos >= VfdCache[file].fileSize )
+ {
+ temporary_files_size += (double)(VfdCache[file].seekPos - VfdCache[file].fileSize);
+ VfdCache[file].fileSize = VfdCache[file].seekPos;
+
+ if (temporary_files_size / 1024.0 > (double)temp_file_limit)
+ ereport(ERROR,
+ (errcode(ERRCODE_QUERY_CANCELED),
+ errmsg("aborting due to exceeding temp file limit")));
+ }
+ }
+ }
else
{
/*
@@ -1887,6 +1921,7 @@ CleanupTempFiles(bool isProcExit)
}
have_pending_fd_cleanup = false;
+ temporary_files_size = 0.0;
}
/* Clean up "allocated" stdio files and dirs. */
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 92391ed..17c062e 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -423,6 +423,7 @@ int log_min_messages = WARNING;
int client_min_messages = NOTICE;
int log_min_duration_statement = -1;
int log_temp_files = -1;
+int temp_file_limit = -1;
int trace_recovery_messages = LOG;
int num_temp_buffers = 1024;
@@ -535,6 +536,8 @@ const char *const config_group_names[] =
gettext_noop("Resource Usage"),
/* RESOURCES_MEM */
gettext_noop("Resource Usage / Memory"),
+ /* RESOURCES_DISK */
+ gettext_noop("Resource Usage / Disk"),
/* RESOURCES_KERNEL */
gettext_noop("Resource Usage / Kernel Resources"),
/* RESOURCES_VACUUM_DELAY */
@@ -2185,6 +2188,15 @@ static struct config_int ConfigureNamesInt[] =
RELSEG_SIZE, RELSEG_SIZE, RELSEG_SIZE,
NULL, NULL, NULL
},
+ {
+ {"temp_file_limit", PGC_SUSET, RESOURCES_DISK,
+ gettext_noop("Size of all temp files."),
+ gettext_noop("-1 is ignored >=0 means enforce size limit."),
+ GUC_UNIT_KB
+ },
+ &temp_file_limit,
+ -1, -1, MAX_KILOBYTES, NULL, NULL
+ },
{
{"wal_block_size", PGC_INTERNAL, PRESET_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 655dad4..cc468c7 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -118,6 +118,7 @@
#work_mem = 1MB # min 64kB
#maintenance_work_mem = 16MB # min 1MB
#max_stack_depth = 2MB # min 100kB
+#temp_file_limit = -1 # limit backend temp file space
# - Kernel Resource Usage -
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index ee52cd7..35321ee 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -208,6 +208,7 @@ extern int log_min_messages;
extern int client_min_messages;
extern int log_min_duration_statement;
extern int log_temp_files;
+extern int temp_file_limit;
extern int num_temp_buffers;
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index a1ca012..3d9ab37 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -59,6 +59,7 @@ enum config_group
CONN_AUTH_SECURITY,
RESOURCES,
RESOURCES_MEM,
+ RESOURCES_DISK,
RESOURCES_KERNEL,
RESOURCES_VACUUM_DELAY,
RESOURCES_BGWRITER,
diff --git a/src/test/regress/expected/resource.out b/src/test/regress/expected/resource.out
new file mode 100644
index 0000000..571fa8f
--- /dev/null
+++ b/src/test/regress/expected/resource.out
@@ -0,0 +1,18 @@
+--
+-- RESOURCE
+-- Test resource management capabilities
+--
+-- temp_file_limit
+CREATE TEMPORARY TABLE resourcetemp1 AS SELECT generate_series(1,100000);
+-- Should succeed
+SET temp_file_limit = 10000;
+SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a;
+ count
+--------
+ 100000
+(1 row)
+
+-- Should fail
+SET temp_file_limit = 1000;
+SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a;
+ERROR: aborting due to exceeding temp file limit
diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule
index bb654f9..325cb3d 100644
--- a/src/test/regress/serial_schedule
+++ b/src/test/regress/serial_schedule
@@ -127,3 +127,4 @@ test: largeobject
test: with
test: xml
test: stats
+test: resource
diff --git a/src/test/regress/sql/resource.sql b/src/test/regress/sql/resource.sql
new file mode 100644
index 0000000..c593a58
--- /dev/null
+++ b/src/test/regress/sql/resource.sql
@@ -0,0 +1,17 @@
+--
+-- RESOURCE
+-- Test resource management capabilities
+--
+
+-- temp_file_limit
+CREATE TEMPORARY TABLE resourcetemp1 AS SELECT generate_series(1,100000);
+
+-- Should succeed
+SET temp_file_limit = 10000;
+
+SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a;
+
+-- Should fail
+SET temp_file_limit = 1000;
+
+SELECT count(*) FROM (select * FROM resourcetemp1 ORDER BY 1) AS a;
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
