On Fri, Jun 19, 2015 at 9:22 PM, Robert Haas <robertmh...@gmail.com> wrote: > On Fri, Jun 19, 2015 at 8:18 AM, Robert Haas <robertmh...@gmail.com> wrote: >> On Fri, Jun 19, 2015 at 12:14 AM, Michael Paquier >> <michael.paqu...@gmail.com> wrote: >>>> Listing the directories with pg_ls_dir() has the same problem. >>> >>> (After some discussion on IM with Heikki on this one). >>> This is actually more tricky because pg_ls_dir() does not return '.' >>> or '..' that we could use to identify that the directory actually >>> exists or not when it is empty. Hence I think that we should add two >>> options to pg_ls_dir: >>> - include_self, default to false. If set to true, '.' is added in the >>> list of items. >>> - if_not_exists, to bypass error that a folder does not exist, default >>> at false. If if_not_exists = true and include_self = true, returning >>> only '.' would mean that the folder exist but that it is empty. If >>> if_not_exists = true and include_self = false, no rows are returned. >>> We could as well ERROR as well if both options are set like that. I am >>> fine with any of them as long as behavior is properly documented. >> >> Including '.' to distinguish between an empty directory and a >> nonexistent one seems like an unnecessarily complicated and >> non-obvious API. How about just one additional parameter bool >> *exists. If NULL and no directory, ERROR, else on return set *exists >> to true or false. > > Err, wait. You're talking about an SQL function, heh heh. So that > won't work. Maybe what you proposed is the best we can do, then, > although I would suggest that you rename the include_self parameter to > something like include_dot_dirs and return both "." and "..". > Returning only "." seems like it will seem weird to people.
So... Attached are a set of patches dedicated at fixing this issue: - 0001, add if_not_exists to pg_tablespace_location, returning NULL if path does not exist - 0002, same with pg_stat_file, returning NULL if file does not exist - 0003, same with pg_read_*file. I added them to all the existing functions for consistency. - 0004, pg_ls_dir extended with if_not_exists and include_dot_dirs (thanks Robert for the naming!) - 0005, as things get complex, a set of regression tests aimed to covering those things. pg_tablespace_location is platform-dependent, so there are no tests for it. - 0006, the fix for pg_rewind, using what has been implemented before. Attached is an updated test case triggering the issue (rewind_test.bash), with the small patch attached that adds a pg_sleep call in pg_rewind.c (20150623_pg_rewind_sleep.patch). I imagine that this is a bug people are going to meet in the field easily, particularly with temporary relation files or temporary XLOG files. Regards, -- Michael
From 454fa9e67c67621b0832d7fbd90153328c99197b Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Tue, 23 Jun 2015 11:45:49 +0900
Subject: [PATCH 1/6] Extend pg_tablespace_location with if_not_exists option
This is useful for code paths that prefer receive an empty response
instead of an ERROR if tablespace specified does not exist.
---
doc/src/sgml/func.sgml | 8 ++++--
src/backend/utils/adt/misc.c | 60 ++++++++++++++++++++++++++++++++++---------
src/include/catalog/pg_proc.h | 2 ++
src/include/utils/builtins.h | 1 +
4 files changed, 57 insertions(+), 14 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 650051b..7929e7e 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -15844,9 +15844,13 @@ SELECT pg_type_is_visible('myschema.widget'::regtype);
<entry>get the set of database OIDs that have objects in the tablespace</entry>
</row>
<row>
- <entry><literal><function>pg_tablespace_location(<parameter>tablespace_oid</parameter>)</function></literal></entry>
+ <entry><literal><function>pg_tablespace_location(<parameter>tablespace_oid</parameter> [, <parameter>if_not_exists</> <type>boolean</type>])</function></literal></entry>
<entry><type>text</type></entry>
- <entry>get the path in the file system that this tablespace is located in</entry>
+ <entry>
+ Get the path in the file system that this tablespace is located in.
+ If <parameter>if_not_exists</parameter> is true, <literal>NULL</literal>
+ is returned if tablespace does not exist instead of an error.
+ </entry>
</row>
<row>
<entry><literal><function>pg_typeof(<parameter>any</parameter>)</function></literal></entry>
diff --git a/src/backend/utils/adt/misc.c b/src/backend/utils/adt/misc.c
index c0495d9..034728d 100644
--- a/src/backend/utils/adt/misc.c
+++ b/src/backend/utils/adt/misc.c
@@ -336,17 +336,24 @@ pg_tablespace_databases(PG_FUNCTION_ARGS)
SRF_RETURN_DONE(funcctx);
}
-
/*
- * pg_tablespace_location - get location for a tablespace
+ * Wrapper function for pg_tablespace_location and
+ * pg_tablespace_location_extended.
*/
-Datum
-pg_tablespace_location(PG_FUNCTION_ARGS)
+static Datum
+tablespace_location_wrapper(FunctionCallInfo fcinfo)
{
Oid tablespaceOid = PG_GETARG_OID(0);
char sourcepath[MAXPGPATH];
char targetpath[MAXPGPATH];
int rllen;
+ bool if_not_exists = false;
+
+ /*
+ * Check for IF NOT EXISTS option.
+ */
+ if (PG_NARGS() == 2 && !PG_ARGISNULL(1))
+ if_not_exists = PG_GETARG_BOOL(1);
/*
* It's useful to apply this function to pg_class.reltablespace, wherein
@@ -373,15 +380,25 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
rllen = readlink(sourcepath, targetpath, sizeof(targetpath));
if (rllen < 0)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not read symbolic link \"%s\": %m",
- sourcepath)));
+ {
+ if (!if_not_exists)
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not read symbolic link \"%s\": %m",
+ sourcepath)));
+ else
+ PG_RETURN_NULL();
+ }
if (rllen >= sizeof(targetpath))
- ereport(ERROR,
- (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
- errmsg("symbolic link \"%s\" target is too long",
- sourcepath)));
+ {
+ if (!if_not_exists)
+ ereport(ERROR,
+ (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
+ errmsg("symbolic link \"%s\" target is too long",
+ sourcepath)));
+ else
+ PG_RETURN_NULL();
+ }
targetpath[rllen] = '\0';
PG_RETURN_TEXT_P(cstring_to_text(targetpath));
@@ -394,6 +411,25 @@ pg_tablespace_location(PG_FUNCTION_ARGS)
}
/*
+ * pg_tablespace_location - get location for a tablespace
+ */
+Datum
+pg_tablespace_location(PG_FUNCTION_ARGS)
+{
+ return tablespace_location_wrapper(fcinfo);
+}
+
+/*
+ * pg_tablespace_location - get location for a tablespace, with option to
+ * bypass errors in case of a non-existing tablespace.
+ */
+Datum
+pg_tablespace_location_extended(PG_FUNCTION_ARGS)
+{
+ return tablespace_location_wrapper(fcinfo);
+}
+
+/*
* pg_sleep - delay for N seconds
*/
Datum
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index 6b3d194..1f163f0 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -2922,6 +2922,8 @@ DESCR("current trigger depth");
DATA(insert OID = 3778 ( pg_tablespace_location PGNSP PGUID 12 1 0 0 0 f f f f t f s 1 0 25 "26" _null_ _null_ _null_ _null_ _null_ pg_tablespace_location _null_ _null_ _null_ ));
DESCR("tablespace location");
+DATA(insert OID = 3579 ( pg_tablespace_location PGNSP PGUID 12 1 0 0 0 f f f f t f s 2 0 25 "26 16" _null_ _null_ _null_ _null_ _null_ pg_tablespace_location_extended _null_ _null_ _null_ ));
+DESCR("tablespace location");
DATA(insert OID = 1946 ( encode PGNSP PGUID 12 1 0 0 0 f f f f t f i 2 0 25 "17 25" _null_ _null_ _null_ _null_ _null_ binary_encode _null_ _null_ _null_ ));
DESCR("convert bytea value into some ascii-only text string");
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 51f25a2..ab78ec2 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -486,6 +486,7 @@ extern Datum pg_terminate_backend(PG_FUNCTION_ARGS);
extern Datum pg_reload_conf(PG_FUNCTION_ARGS);
extern Datum pg_tablespace_databases(PG_FUNCTION_ARGS);
extern Datum pg_tablespace_location(PG_FUNCTION_ARGS);
+extern Datum pg_tablespace_location_extended(PG_FUNCTION_ARGS);
extern Datum pg_rotate_logfile(PG_FUNCTION_ARGS);
extern Datum pg_sleep(PG_FUNCTION_ARGS);
extern Datum pg_get_keywords(PG_FUNCTION_ARGS);
--
2.4.4
From 72ef91958e91e9d6ba4a72d325872edb65166cc7 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Tue, 23 Jun 2015 11:48:34 +0900
Subject: [PATCH 2/6] Extend pg_stat_file with if_not_exists option
This is useful for code paths that prefer receiving a NULL result
instead of an ERROR if file specified does not exist.
---
doc/src/sgml/func.sgml | 9 ++++--
src/backend/utils/adt/genfile.c | 65 +++++++++++++++++++++++++++++++----------
src/include/catalog/pg_proc.h | 2 ++
src/include/utils/builtins.h | 1 +
4 files changed, 59 insertions(+), 18 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7929e7e..0f70985 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17830,10 +17830,15 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
</row>
<row>
<entry>
- <literal><function>pg_stat_file(<parameter>filename</> <type>text</>)</function></literal>
+ <literal><function>pg_stat_file(<parameter>filename</> <type>text</>[, <parameter>if_not_exists</> <type>boolean</type>])</function></literal>
</entry>
<entry><type>record</type></entry>
- <entry>Return information about a file</entry>
+ <entry>
+ Return information about a file. If
+ <parameter>if_not_exists</parameter> is true, <literal>NULL</literal>
+ is returned for all result fields if file does not exist instead of
+ an error.
+ </entry>
</row>
</tbody>
</tgroup>
diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c
index f3f3cca..93eb379 100644
--- a/src/backend/utils/adt/genfile.c
+++ b/src/backend/utils/adt/genfile.c
@@ -254,10 +254,10 @@ pg_read_binary_file_all(PG_FUNCTION_ARGS)
}
/*
- * stat a file
+ * Wrapper for pg_stat_file and pg_stat_file_extended.
*/
-Datum
-pg_stat_file(PG_FUNCTION_ARGS)
+static Datum
+stat_file_wrapper(PG_FUNCTION_ARGS)
{
text *filename_t = PG_GETARG_TEXT_P(0);
char *filename;
@@ -266,18 +266,29 @@ pg_stat_file(PG_FUNCTION_ARGS)
bool isnull[6];
HeapTuple tuple;
TupleDesc tupdesc;
+ bool if_not_exists = false;
+ bool return_null = false;
if (!superuser())
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser to get file information"))));
+ /* Check for option if_not_exists */
+ if (PG_NARGS() == 2 && !PG_ARGISNULL(1))
+ if_not_exists = PG_GETARG_BOOL(1);
+
filename = convert_and_check_filename(filename_t);
if (stat(filename, &fst) < 0)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not stat file \"%s\": %m", filename)));
+ {
+ if (if_not_exists)
+ return_null = true;
+ else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", filename)));
+ }
/*
* This record type had better match the output parameters declared for me
@@ -298,20 +309,23 @@ pg_stat_file(PG_FUNCTION_ARGS)
"isdir", BOOLOID, -1, 0);
BlessTupleDesc(tupdesc);
- memset(isnull, false, sizeof(isnull));
+ memset(isnull, return_null, sizeof(isnull));
- values[0] = Int64GetDatum((int64) fst.st_size);
- values[1] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_atime));
- values[2] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_mtime));
- /* Unix has file status change time, while Win32 has creation time */
+ if (!return_null)
+ {
+ values[0] = Int64GetDatum((int64) fst.st_size);
+ values[1] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_atime));
+ values[2] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_mtime));
+ /* Unix has file status change time, while Win32 has creation time */
#if !defined(WIN32) && !defined(__CYGWIN__)
- values[3] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime));
- isnull[4] = true;
+ values[3] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime));
+ isnull[4] = true;
#else
- isnull[3] = true;
- values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime));
+ isnull[3] = true;
+ values[4] = TimestampTzGetDatum(time_t_to_timestamptz(fst.st_ctime));
#endif
- values[5] = BoolGetDatum(S_ISDIR(fst.st_mode));
+ values[5] = BoolGetDatum(S_ISDIR(fst.st_mode));
+ }
tuple = heap_form_tuple(tupdesc, values, isnull);
@@ -320,6 +334,25 @@ pg_stat_file(PG_FUNCTION_ARGS)
PG_RETURN_DATUM(HeapTupleGetDatum(tuple));
}
+/*
+ * pg_stat_file_extended
+ * Stat a file, with possibility to bypass error in case of a missing file.
+ */
+Datum
+pg_stat_file_extended(PG_FUNCTION_ARGS)
+{
+ return stat_file_wrapper(fcinfo);
+}
+
+/*
+ * stat a file
+ */
+Datum
+pg_stat_file(PG_FUNCTION_ARGS)
+{
+ return stat_file_wrapper(fcinfo);
+}
+
/*
* List a directory (returns the filenames only)
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index 1f163f0..8da1781 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -3185,6 +3185,8 @@ DESCR("rotate log file");
DATA(insert OID = 2623 ( pg_stat_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 2249 "25" "{25,20,1184,1184,1184,1184,16}" "{i,o,o,o,o,o,o}" "{filename,size,access,modification,change,creation,isdir}" _null_ _null_ pg_stat_file _null_ _null_ _null_ ));
DESCR("get information about file");
+DATA(insert OID = 3307 ( pg_stat_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 2 0 2249 "25 16" "{25,16,20,1184,1184,1184,1184,16}" "{i,i,o,o,o,o,o,o}" "{filename,if_not_exists,size,access,modification,change,creation,isdir}" _null_ _null_ pg_stat_file_extended _null_ _null_ _null_ ));
+DESCR("get information about file");
DATA(insert OID = 2624 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 3 0 25 "25 20 20" _null_ _null_ _null_ _null_ _null_ pg_read_file _null_ _null_ _null_ ));
DESCR("read text from a file");
DATA(insert OID = 3826 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ pg_read_file_all _null_ _null_ _null_ ));
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index ab78ec2..e46650a 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -472,6 +472,7 @@ extern Datum pg_relation_filepath(PG_FUNCTION_ARGS);
extern bytea *read_binary_file(const char *filename,
int64 seek_offset, int64 bytes_to_read);
extern Datum pg_stat_file(PG_FUNCTION_ARGS);
+extern Datum pg_stat_file_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_file(PG_FUNCTION_ARGS);
extern Datum pg_read_file_all(PG_FUNCTION_ARGS);
extern Datum pg_read_binary_file(PG_FUNCTION_ARGS);
--
2.4.4
From b4e067ccefef8aac912211aee792af91c624b553 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Tue, 23 Jun 2015 11:51:30 +0900
Subject: [PATCH 3/6] Add IF NOT EXISTS to pg_read_file and pg_read_binary_file
This is useful for code paths that do not want to fail if the file
queried does not exist.
---
doc/src/sgml/func.sgml | 16 +++-
src/backend/commands/extension.c | 2 +-
src/backend/utils/adt/genfile.c | 174 +++++++++++++++++++++++++++++++++------
src/include/catalog/pg_proc.h | 8 ++
src/include/utils/builtins.h | 8 +-
5 files changed, 179 insertions(+), 29 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 0f70985..edfea8b 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17816,17 +17816,25 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
</row>
<row>
<entry>
- <literal><function>pg_read_file(<parameter>filename</> <type>text</> [, <parameter>offset</> <type>bigint</>, <parameter>length</> <type>bigint</>])</function></literal>
+ <literal><function>pg_read_file(<parameter>filename</> <type>text</> [, <parameter>offset</> <type>bigint</>, <parameter>length</> <type>bigint</> [, <parameter>if_not_exists</> <type>boolean</>] ])</function></literal>
</entry>
<entry><type>text</type></entry>
- <entry>Return the contents of a text file</entry>
+ <entry>
+ Return the contents of a text file. If
+ <parameter>if_not_exists</parameter> is true, <literal>NULL</literal>
+ is returned if file does not exist instead of an error.
+ </entry>
</row>
<row>
<entry>
- <literal><function>pg_read_binary_file(<parameter>filename</> <type>text</> [, <parameter>offset</> <type>bigint</>, <parameter>length</> <type>bigint</>])</function></literal>
+ <literal><function>pg_read_binary_file(<parameter>filename</> <type>text</> [, <parameter>offset</> <type>bigint</>, <parameter>length</> <type>bigint</> [, <parameter>if_not_exists</> <type>boolean</>] ])</function></literal>
</entry>
<entry><type>bytea</type></entry>
- <entry>Return the contents of a file</entry>
+ <entry>
+ Return the contents of a file. If
+ <parameter>if_not_exists</parameter> is true, <literal>NULL</literal>
+ is returned if file does not exist instead of an error.
+ </entry>
</row>
<row>
<entry>
diff --git a/src/backend/commands/extension.c b/src/backend/commands/extension.c
index 5cc74d0..25bbe01 100644
--- a/src/backend/commands/extension.c
+++ b/src/backend/commands/extension.c
@@ -640,7 +640,7 @@ read_extension_script_file(const ExtensionControlFile *control,
char *dest_str;
int len;
- content = read_binary_file(filename, 0, -1);
+ content = read_binary_file(filename, 0, -1, false);
/* use database encoding if not given */
if (control->encoding < 0)
diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c
index 93eb379..6ce9ec3 100644
--- a/src/backend/utils/adt/genfile.c
+++ b/src/backend/utils/adt/genfile.c
@@ -88,7 +88,8 @@ convert_and_check_filename(text *arg)
* We read the whole of the file when bytes_to_read is negative.
*/
bytea *
-read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
+read_binary_file(const char *filename, int64 seek_offset,
+ int64 bytes_to_read, bool if_not_exists)
{
bytea *buf;
size_t nbytes;
@@ -103,9 +104,14 @@ read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
struct stat fst;
if (stat(filename, &fst) < 0)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not stat file \"%s\": %m", filename)));
+ {
+ if (if_not_exists)
+ return NULL;
+ else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not stat file \"%s\": %m", filename)));
+ }
bytes_to_read = fst.st_size - seek_offset;
}
@@ -118,10 +124,15 @@ read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
errmsg("requested length too large")));
if ((file = AllocateFile(filename, PG_BINARY_R)) == NULL)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not open file \"%s\" for reading: %m",
- filename)));
+ {
+ if (if_not_exists)
+ return NULL;
+ else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open file \"%s\" for reading: %m",
+ filename)));
+ }
if (fseeko(file, (off_t) seek_offset,
(seek_offset >= 0) ? SEEK_SET : SEEK_END) != 0)
@@ -150,11 +161,16 @@ read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
* in the database encoding.
*/
static text *
-read_text_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
+read_text_file(const char *filename, int64 seek_offset,
+ int64 bytes_to_read, bool if_not_exists)
{
bytea *buf;
- buf = read_binary_file(filename, seek_offset, bytes_to_read);
+ buf = read_binary_file(filename, seek_offset,
+ bytes_to_read, if_not_exists);
+
+ if (buf == NULL)
+ return NULL;
/* Make sure the input is valid */
pg_verifymbstr(VARDATA(buf), VARSIZE(buf) - VARHDRSZ, false);
@@ -164,21 +180,27 @@ read_text_file(const char *filename, int64 seek_offset, int64 bytes_to_read)
}
/*
- * Read a section of a file, returning it as text
+ * Wrapper function for pg_read_file and pg_read_file_extended.
*/
-Datum
-pg_read_file(PG_FUNCTION_ARGS)
+static Datum
+read_file_wrapper(PG_FUNCTION_ARGS)
{
text *filename_t = PG_GETARG_TEXT_P(0);
int64 seek_offset = PG_GETARG_INT64(1);
int64 bytes_to_read = PG_GETARG_INT64(2);
+ bool if_not_exists = false;
char *filename;
+ bytea *result;
if (!superuser())
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser to read files"))));
+ /* Check for option if_not_exists */
+ if (PG_NARGS() == 4 && !PG_ARGISNULL(3))
+ if_not_exists = PG_GETARG_BOOL(3);
+
filename = convert_and_check_filename(filename_t);
if (bytes_to_read < 0)
@@ -186,44 +208,100 @@ pg_read_file(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("requested length cannot be negative")));
- PG_RETURN_TEXT_P(read_text_file(filename, seek_offset, bytes_to_read));
+ result = read_text_file(filename, seek_offset, bytes_to_read,
+ if_not_exists);
+ if (result == NULL)
+ PG_RETURN_NULL();
+ PG_RETURN_TEXT_P(result);
}
/*
- * Read the whole of a file, returning it as text
+ * Read a section of a file, returning it as text
*/
Datum
-pg_read_file_all(PG_FUNCTION_ARGS)
+pg_read_file(PG_FUNCTION_ARGS)
+{
+ return read_file_wrapper(fcinfo);
+}
+
+/*
+ * Similar to pg_read_file, with IF NOT EXISTS option.
+ */
+Datum
+pg_read_file_extended(PG_FUNCTION_ARGS)
+{
+ return read_file_wrapper(fcinfo);
+}
+
+/*
+ * Wrapper function for pg_read_file_all and pg_read_file_all_extended.
+ */
+static Datum
+read_file_all_wrapper(PG_FUNCTION_ARGS)
{
text *filename_t = PG_GETARG_TEXT_P(0);
char *filename;
+ text *result;
+ bool if_not_exists = false;
if (!superuser())
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser to read files"))));
+ /* Check for option if_not_exists */
+ if (PG_NARGS() == 2 && !PG_ARGISNULL(1))
+ if_not_exists = PG_GETARG_BOOL(1);
+
filename = convert_and_check_filename(filename_t);
- PG_RETURN_TEXT_P(read_text_file(filename, 0, -1));
+ result = read_text_file(filename, 0, -1, if_not_exists);
+ if (result == NULL)
+ PG_RETURN_NULL();
+
+ PG_RETURN_TEXT_P(result);
}
/*
- * Read a section of a file, returning it as bytea
+ * Read the whole of a file, returning it as text
*/
Datum
-pg_read_binary_file(PG_FUNCTION_ARGS)
+pg_read_file_all(PG_FUNCTION_ARGS)
+{
+ return read_file_all_wrapper(fcinfo);
+}
+
+/*
+ * Similar to pg_read_file_all, with IF NOT EXISTS option.
+ */
+Datum
+pg_read_file_all_extended(PG_FUNCTION_ARGS)
+{
+ return read_file_all_wrapper(fcinfo);
+}
+
+/*
+ * Wrapper for pg_read_binary_file and pg_read_binary_file_extended.
+ */
+static Datum
+read_binary_file_wrapper(PG_FUNCTION_ARGS)
{
text *filename_t = PG_GETARG_TEXT_P(0);
int64 seek_offset = PG_GETARG_INT64(1);
int64 bytes_to_read = PG_GETARG_INT64(2);
char *filename;
+ bool if_not_exists = false;
+ bytea *result;
if (!superuser())
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser to read files"))));
+ /* Check for option if_not_exists */
+ if (PG_NARGS() == 4 && !PG_ARGISNULL(3))
+ if_not_exists = PG_GETARG_BOOL(3);
+
filename = convert_and_check_filename(filename_t);
if (bytes_to_read < 0)
@@ -231,26 +309,76 @@ pg_read_binary_file(PG_FUNCTION_ARGS)
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("requested length cannot be negative")));
- PG_RETURN_BYTEA_P(read_binary_file(filename, seek_offset, bytes_to_read));
+ result = read_binary_file(filename, seek_offset,
+ bytes_to_read, if_not_exists);
+ if (result == NULL)
+ PG_RETURN_NULL();
+ PG_RETURN_BYTEA_P(result);
}
/*
- * Read the whole of a file, returning it as bytea
+ * Read a section of a file, returning it as bytea
*/
Datum
-pg_read_binary_file_all(PG_FUNCTION_ARGS)
+pg_read_binary_file(PG_FUNCTION_ARGS)
+{
+ return read_binary_file_wrapper(fcinfo);
+}
+
+/*
+ * Similar to pg_read_binary_file, with IF NOT EXISTS option.
+ */
+Datum
+pg_read_binary_file_extended(PG_FUNCTION_ARGS)
+{
+ return read_binary_file_wrapper(fcinfo);
+}
+
+/*
+ * Wrapper for pg_read_binary_file_all and pg_read_binary_file_all_extended.
+ */
+static Datum
+read_binary_file_all_wrapper(PG_FUNCTION_ARGS)
{
text *filename_t = PG_GETARG_TEXT_P(0);
char *filename;
+ bool if_not_exists = false;
+ text *result;
if (!superuser())
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
(errmsg("must be superuser to read files"))));
+ /* Check for option if_not_exists */
+ if (PG_NARGS() == 2 && !PG_ARGISNULL(1))
+ if_not_exists = PG_GETARG_BOOL(1);
+
filename = convert_and_check_filename(filename_t);
- PG_RETURN_BYTEA_P(read_binary_file(filename, 0, -1));
+ result = read_binary_file(filename, 0, -1, if_not_exists);
+ if (result == NULL)
+ PG_RETURN_NULL();
+
+ PG_RETURN_BYTEA_P(result);
+}
+
+/*
+ * Read the whole of a file, returning it as bytea
+ */
+Datum
+pg_read_binary_file_all(PG_FUNCTION_ARGS)
+{
+ return read_binary_file_all_wrapper(fcinfo);
+}
+
+/*
+ * Similar to pg_read_binary_file_all, with IF NOT EXISTS option.
+ */
+Datum
+pg_read_binary_file_all_extended(PG_FUNCTION_ARGS)
+{
+ return read_binary_file_all_wrapper(fcinfo);
}
/*
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index 8da1781..c805a9a 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -3189,12 +3189,20 @@ DATA(insert OID = 3307 ( pg_stat_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 2 0
DESCR("get information about file");
DATA(insert OID = 2624 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 3 0 25 "25 20 20" _null_ _null_ _null_ _null_ _null_ pg_read_file _null_ _null_ _null_ ));
DESCR("read text from a file");
+DATA(insert OID = 3293 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 4 0 25 "25 20 20 16" _null_ _null_ _null_ _null_ _null_ pg_read_file_extended _null_ _null_ _null_ ));
+DESCR("read text from a file");
DATA(insert OID = 3826 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ pg_read_file_all _null_ _null_ _null_ ));
DESCR("read text from a file");
+DATA(insert OID = 3294 ( pg_read_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 2 0 25 "25 16" _null_ _null_ _null_ _null_ _null_ pg_read_file_all_extended _null_ _null_ _null_ ));
+DESCR("read text from a file");
DATA(insert OID = 3827 ( pg_read_binary_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 3 0 17 "25 20 20" _null_ _null_ _null_ _null_ _null_ pg_read_binary_file _null_ _null_ _null_ ));
DESCR("read bytea from a file");
+DATA(insert OID = 3295 ( pg_read_binary_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 4 0 17 "25 20 20 16" _null_ _null_ _null_ _null_ _null_ pg_read_binary_file_extended _null_ _null_ _null_ ));
+DESCR("read bytea from a file");
DATA(insert OID = 3828 ( pg_read_binary_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 17 "25" _null_ _null_ _null_ _null_ _null_ pg_read_binary_file_all _null_ _null_ _null_ ));
DESCR("read bytea from a file");
+DATA(insert OID = 3296 ( pg_read_binary_file PGNSP PGUID 12 1 0 0 0 f f f f t f v 2 0 17 "25 16" _null_ _null_ _null_ _null_ _null_ pg_read_binary_file_all_extended _null_ _null_ _null_ ));
+DESCR("read bytea from a file");
DATA(insert OID = 2625 ( pg_ls_dir PGNSP PGUID 12 1 1000 0 0 f f f f t t v 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ pg_ls_dir _null_ _null_ _null_ ));
DESCR("list all files in a directory");
DATA(insert OID = 2626 ( pg_sleep PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 2278 "701" _null_ _null_ _null_ _null_ _null_ pg_sleep _null_ _null_ _null_ ));
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index e46650a..82d0e18 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -470,13 +470,19 @@ extern Datum pg_relation_filepath(PG_FUNCTION_ARGS);
/* genfile.c */
extern bytea *read_binary_file(const char *filename,
- int64 seek_offset, int64 bytes_to_read);
+ int64 seek_offset,
+ int64 bytes_to_read,
+ bool if_not_exists);
extern Datum pg_stat_file(PG_FUNCTION_ARGS);
extern Datum pg_stat_file_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_file(PG_FUNCTION_ARGS);
+extern Datum pg_read_file_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_file_all(PG_FUNCTION_ARGS);
+extern Datum pg_read_file_all_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_binary_file(PG_FUNCTION_ARGS);
+extern Datum pg_read_binary_file_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_binary_file_all(PG_FUNCTION_ARGS);
+extern Datum pg_read_binary_file_all_extended(PG_FUNCTION_ARGS);
extern Datum pg_ls_dir(PG_FUNCTION_ARGS);
/* misc.c */
--
2.4.4
From f40a675db5e086a72e52cc9d0394728a0ada95f8 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Mon, 22 Jun 2015 21:19:33 +0900
Subject: [PATCH 4/6] Extend pg_ls_dir with include_dot_dirs and if_not_exists
If include_dot_dirs is set to true, "." and ".." are listed in the list
of files listed. If if_not_exists is true, an empty list of files is
returned to caller instead of erroring out if the folder specified does
not exist.
---
doc/src/sgml/func.sgml | 10 +++++--
src/backend/utils/adt/genfile.c | 59 +++++++++++++++++++++++++++++++++--------
src/include/catalog/pg_proc.h | 2 ++
src/include/utils/builtins.h | 1 +
4 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index edfea8b..e9dc19f 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -17809,10 +17809,16 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
<tbody>
<row>
<entry>
- <literal><function>pg_ls_dir(<parameter>dirname</> <type>text</>)</function></literal>
+ <literal><function>pg_ls_dir(<parameter>dirname</> <type>text</> [, <parameter>if_not_exists</> <type>boolean</>, <parameter>include_dot_dirs</> <type>boolean</>] )</function></literal>
</entry>
<entry><type>setof text</type></entry>
- <entry>List the contents of a directory</entry>
+ <entry>
+ List the contents of a directory. If
+ <parameter>if_not_exists</parameter> is true, an empty result is
+ returned instead of an error. If <parameter>include_dot_dirs</>
+ is true, <quote>.</> and <quote>..</> are included as well in
+ the returned list.
+ </entry>
</row>
<row>
<entry>
diff --git a/src/backend/utils/adt/genfile.c b/src/backend/utils/adt/genfile.c
index 6ce9ec3..6af7253 100644
--- a/src/backend/utils/adt/genfile.c
+++ b/src/backend/utils/adt/genfile.c
@@ -35,6 +35,7 @@ typedef struct
{
char *location;
DIR *dirdesc;
+ bool include_dot_dirs;
} directory_fctx;
@@ -483,14 +484,15 @@ pg_stat_file(PG_FUNCTION_ARGS)
/*
- * List a directory (returns the filenames only)
+ * Wrapper for pg_ls_dir and pg_ls_dir_extended.
*/
-Datum
-pg_ls_dir(PG_FUNCTION_ARGS)
+static Datum
+ls_dir_wrapper(PG_FUNCTION_ARGS)
{
FuncCallContext *funcctx;
struct dirent *de;
directory_fctx *fctx;
+ MemoryContext oldcontext;
if (!superuser())
ereport(ERROR,
@@ -499,7 +501,17 @@ pg_ls_dir(PG_FUNCTION_ARGS)
if (SRF_IS_FIRSTCALL())
{
- MemoryContext oldcontext;
+ bool if_not_exists = false;
+ bool include_dot_dirs = false;
+
+ /* see if custom parameters have been passed down */
+ if (PG_NARGS() == 3)
+ {
+ if (!PG_ARGISNULL(1))
+ if_not_exists = PG_GETARG_BOOL(1);
+ if (!PG_ARGISNULL(2))
+ include_dot_dirs = PG_GETARG_BOOL(2);
+ }
funcctx = SRF_FIRSTCALL_INIT();
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
@@ -508,13 +520,18 @@ pg_ls_dir(PG_FUNCTION_ARGS)
fctx->location = convert_and_check_filename(PG_GETARG_TEXT_P(0));
fctx->dirdesc = AllocateDir(fctx->location);
+ fctx->include_dot_dirs = include_dot_dirs;
if (!fctx->dirdesc)
- ereport(ERROR,
- (errcode_for_file_access(),
- errmsg("could not open directory \"%s\": %m",
- fctx->location)));
-
+ {
+ if (if_not_exists && errno == ENOENT)
+ SRF_RETURN_DONE(funcctx);
+ else
+ ereport(ERROR,
+ (errcode_for_file_access(),
+ errmsg("could not open directory \"%s\": %m",
+ fctx->location)));
+ }
funcctx->user_fctx = fctx;
MemoryContextSwitchTo(oldcontext);
}
@@ -524,8 +541,9 @@ pg_ls_dir(PG_FUNCTION_ARGS)
while ((de = ReadDir(fctx->dirdesc, fctx->location)) != NULL)
{
- if (strcmp(de->d_name, ".") == 0 ||
- strcmp(de->d_name, "..") == 0)
+ if (!fctx->include_dot_dirs &&
+ (strcmp(de->d_name, ".") == 0 ||
+ strcmp(de->d_name, "..") == 0))
continue;
SRF_RETURN_NEXT(funcctx, CStringGetTextDatum(de->d_name));
@@ -535,3 +553,22 @@ pg_ls_dir(PG_FUNCTION_ARGS)
SRF_RETURN_DONE(funcctx);
}
+
+/*
+ * List a directory (returns the filenames only)
+ */
+Datum
+pg_ls_dir(PG_FUNCTION_ARGS)
+{
+ return ls_dir_wrapper(fcinfo);
+}
+
+/*
+ * List a directory (can optionally return dot directories or
+ * use IF NOT EXISTS).
+ */
+Datum
+pg_ls_dir_extended(PG_FUNCTION_ARGS)
+{
+ return ls_dir_wrapper(fcinfo);
+}
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index c805a9a..b559572 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -3205,6 +3205,8 @@ DATA(insert OID = 3296 ( pg_read_binary_file PGNSP PGUID 12 1 0 0 0 f f f f t f
DESCR("read bytea from a file");
DATA(insert OID = 2625 ( pg_ls_dir PGNSP PGUID 12 1 1000 0 0 f f f f t t v 1 0 25 "25" _null_ _null_ _null_ _null_ _null_ pg_ls_dir _null_ _null_ _null_ ));
DESCR("list all files in a directory");
+DATA(insert OID = 3297 ( pg_ls_dir PGNSP PGUID 12 1 1000 0 0 f f f f t t v 3 0 25 "25 16 16" _null_ _null_ _null_ _null_ _null_ pg_ls_dir_extended _null_ _null_ _null_ ));
+DESCR("list all files in a directory");
DATA(insert OID = 2626 ( pg_sleep PGNSP PGUID 12 1 0 0 0 f f f f t f v 1 0 2278 "701" _null_ _null_ _null_ _null_ _null_ pg_sleep _null_ _null_ _null_ ));
DESCR("sleep for the specified time in seconds");
DATA(insert OID = 3935 ( pg_sleep_for PGNSP PGUID 14 1 0 0 0 f f f f t f v 1 0 2278 "1186" _null_ _null_ _null_ _null_ _null_ "select pg_catalog.pg_sleep(extract(epoch from pg_catalog.clock_timestamp() operator(pg_catalog.+) $1) operator(pg_catalog.-) extract(epoch from pg_catalog.clock_timestamp()))" _null_ _null_ _null_ ));
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 82d0e18..446cda6 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -484,6 +484,7 @@ extern Datum pg_read_binary_file_extended(PG_FUNCTION_ARGS);
extern Datum pg_read_binary_file_all(PG_FUNCTION_ARGS);
extern Datum pg_read_binary_file_all_extended(PG_FUNCTION_ARGS);
extern Datum pg_ls_dir(PG_FUNCTION_ARGS);
+extern Datum pg_ls_dir_extended(PG_FUNCTION_ARGS);
/* misc.c */
extern Datum current_database(PG_FUNCTION_ARGS);
--
2.4.4
From b15d22e38f0a27e0fedf3fb07e26dcb580f42cb9 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Tue, 23 Jun 2015 10:42:48 +0900
Subject: [PATCH 5/6] Add regression tests for pg_ls_dir and
pg_read_[binary_]file
With the new options if_not_exists that have been added, this looks
worth adding.
---
src/test/regress/expected/file.out | 84 ++++++++++++++++++++++++++++++++++++++
src/test/regress/parallel_schedule | 3 ++
src/test/regress/sql/file.sql | 29 +++++++++++++
3 files changed, 116 insertions(+)
create mode 100644 src/test/regress/expected/file.out
create mode 100644 src/test/regress/sql/file.sql
diff --git a/src/test/regress/expected/file.out b/src/test/regress/expected/file.out
new file mode 100644
index 0000000..0ddae30
--- /dev/null
+++ b/src/test/regress/expected/file.out
@@ -0,0 +1,84 @@
+--
+-- FILE
+--
+-- Tests of SQL functions interacting directly with files of PGDATA
+-- pg_read_file
+SELECT pg_read_file('postgresql.auto.conf', 0, 23); -- OK
+ pg_read_file
+-------------------------
+ # Do not edit this file
+(1 row)
+
+SELECT pg_read_file('postgresql_not_exists', 0, 23); -- not OK
+ERROR: could not open file "postgresql_not_exists" for reading: No such file or directory
+SELECT pg_read_file('postgresql_not_exists', 0, 23, false); -- not OK
+ERROR: could not open file "postgresql_not_exists" for reading: No such file or directory
+SELECT pg_read_file('postgresql_not_exists', 0, 23, true); -- OK
+ pg_read_file
+--------------
+
+(1 row)
+
+-- pg_read_binary_file
+SELECT pg_read_binary_file('postgresql.auto.conf', 0, 23); -- OK
+ pg_read_binary_file
+--------------------------------------------------
+ \x2320446f206e6f74206564697420746869732066696c65
+(1 row)
+
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23); -- not OK
+ERROR: could not open file "postgresql_not_exists" for reading: No such file or directory
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23, false); -- not OK
+ERROR: could not open file "postgresql_not_exists" for reading: No such file or directory
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23, true); -- OK
+ pg_read_binary_file
+---------------------
+
+(1 row)
+
+-- pg_ls_dir
+SELECT pg_ls_dir('postgresql.conf'); -- not OK, not a directory
+ERROR: could not open directory "postgresql.conf": Not a directory
+SELECT pg_ls_dir('pg_twophase'); -- empty directory
+ pg_ls_dir
+-----------
+(0 rows)
+
+SELECT pg_ls_dir('pg_twophase', true, true); -- OK, not empty list
+ pg_ls_dir
+-----------
+ .
+ ..
+(2 rows)
+
+SELECT pg_ls_dir('pg_twophase', false, false); -- OK, empty list
+ pg_ls_dir
+-----------
+(0 rows)
+
+SELECT pg_ls_dir('pg_twophase', false, true); -- OK, not empty list
+ pg_ls_dir
+-----------
+ .
+ ..
+(2 rows)
+
+SELECT pg_ls_dir('pg_twophase', true, false); -- OK, empty list
+ pg_ls_dir
+-----------
+(0 rows)
+
+SELECT pg_ls_dir('postgresql_not_exists', false, true); -- not OK
+ERROR: could not open directory "postgresql_not_exists": No such file or directory
+SELECT pg_ls_dir('postgresql_not_exists', false, false); -- not OK
+ERROR: could not open directory "postgresql_not_exists": No such file or directory
+SELECT pg_ls_dir('postgresql_not_exists', true, true); -- OK, empty list
+ pg_ls_dir
+-----------
+(0 rows)
+
+SELECT pg_ls_dir('postgresql_not_exists', true, false); -- OK, empty list
+ pg_ls_dir
+-----------
+(0 rows)
+
diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule
index 91780cd..b1739fa 100644
--- a/src/test/regress/parallel_schedule
+++ b/src/test/regress/parallel_schedule
@@ -108,5 +108,8 @@ test: plancache limit plpgsql copy2 temp domain rangefuncs prepare without_oid c
# event triggers cannot run concurrently with any test that runs DDL
test: event_trigger
+# Files read during this test may be modified in parallel by other tests
+test: file
+
# run stats by itself because its delay may be insufficient under heavy load
test: stats
diff --git a/src/test/regress/sql/file.sql b/src/test/regress/sql/file.sql
new file mode 100644
index 0000000..a0238f2
--- /dev/null
+++ b/src/test/regress/sql/file.sql
@@ -0,0 +1,29 @@
+--
+-- FILE
+--
+
+-- Tests of SQL functions interacting directly with files of PGDATA
+
+-- pg_read_file
+SELECT pg_read_file('postgresql.auto.conf', 0, 23); -- OK
+SELECT pg_read_file('postgresql_not_exists', 0, 23); -- not OK
+SELECT pg_read_file('postgresql_not_exists', 0, 23, false); -- not OK
+SELECT pg_read_file('postgresql_not_exists', 0, 23, true); -- OK
+
+-- pg_read_binary_file
+SELECT pg_read_binary_file('postgresql.auto.conf', 0, 23); -- OK
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23); -- not OK
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23, false); -- not OK
+SELECT pg_read_binary_file('postgresql_not_exists', 0, 23, true); -- OK
+
+-- pg_ls_dir
+SELECT pg_ls_dir('postgresql.conf'); -- not OK, not a directory
+SELECT pg_ls_dir('pg_twophase'); -- empty directory
+SELECT pg_ls_dir('pg_twophase', true, true); -- OK, not empty list
+SELECT pg_ls_dir('pg_twophase', false, false); -- OK, empty list
+SELECT pg_ls_dir('pg_twophase', false, true); -- OK, not empty list
+SELECT pg_ls_dir('pg_twophase', true, false); -- OK, empty list
+SELECT pg_ls_dir('postgresql_not_exists', false, true); -- not OK
+SELECT pg_ls_dir('postgresql_not_exists', false, false); -- not OK
+SELECT pg_ls_dir('postgresql_not_exists', true, true); -- OK, empty list
+SELECT pg_ls_dir('postgresql_not_exists', true, false); -- OK, empty list
--
2.4.4
From f12a2cec0489d6c20ed55665bc6698ac701e5d73 Mon Sep 17 00:00:00 2001
From: Michael Paquier <michael@otacoo.com>
Date: Tue, 23 Jun 2015 13:16:23 +0900
Subject: [PATCH 6/6] Make pg_rewind able to detect deleted files on remote
source
When getting a list of PGDATA files from a remote server with pg_rewind,
it is possible that this list contains temporary files or files that have
been deleted on the remote server after taking the file list, like what
DROP TABLE would do for a relation. This can lead to errors when calling
pg_read_binary_file on an entry that is listed but does not exist anymore,
preventing the rewind from working correctly.
In order to prevent such problems, use the new if_not_exists options
in pg_ls_dir, pg_tablespace_location and pg_read_binary_file to not
fail if a file does not exist anymore and continue process as for
example a relation file removed does not prevent a rewound node from
replaying WAL in recovery, node rewound guaranteed to have a minimum
recovery point set to the current WAL location of the remote node that
is put in sync with.
---
src/bin/pg_rewind/libpq_fetch.c | 41 ++++++++++++++++++++++++++++++-----------
1 file changed, 30 insertions(+), 11 deletions(-)
diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c
index 6ffd24e..2051d30 100644
--- a/src/bin/pg_rewind/libpq_fetch.c
+++ b/src/bin/pg_rewind/libpq_fetch.c
@@ -149,18 +149,24 @@ libpqProcessFileList(void)
sql =
"WITH RECURSIVE files (path, filename, size, isdir) AS (\n"
" SELECT '' AS path, filename, size, isdir FROM\n"
- " (SELECT pg_ls_dir('.') AS filename) AS fn,\n"
- " pg_stat_file(fn.filename) AS this\n"
+ " (SELECT filename\n"
+ " FROM pg_ls_dir('.', true, true) AS filename\n"
+ " WHERE filename NOT IN ('.', '..')) AS fn,\n"
+ " pg_stat_file(fn.filename, true) AS this\n"
+ " WHERE this.size IS NOT NULL\n"
" UNION ALL\n"
" SELECT parent.path || parent.filename || '/' AS path,\n"
" fn, this.size, this.isdir\n"
" FROM files AS parent,\n"
- " pg_ls_dir(parent.path || parent.filename) AS fn,\n"
- " pg_stat_file(parent.path || parent.filename || '/' || fn) AS this\n"
- " WHERE parent.isdir = 't'\n"
- ")\n"
- "SELECT path || filename, size, isdir,\n"
- " pg_tablespace_location(pg_tablespace.oid) AS link_target\n"
+ " pg_ls_dir(parent.path || parent.filename, true, true) AS fn,\n"
+ " pg_stat_file(parent.path || parent.filename || '/' || fn, true)\n"
+ " AS this\n"
+ " WHERE parent.isdir = 't' AND\n"
+ " (fn NOT IN ('.', '..')) AND\n"
+ " this.size IS NOT NULL)\n"
+ "SELECT path || filename AS file_path, size, isdir,\n"
+ " pg_tablespace_location(pg_tablespace.oid, true)\n"
+ " AS link_target\n"
"FROM files\n"
"LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n"
" AND oid::text = files.filename\n";
@@ -259,8 +265,7 @@ receiveFileChunks(const char *sql)
}
if (PQgetisnull(res, 0, 0) ||
- PQgetisnull(res, 0, 1) ||
- PQgetisnull(res, 0, 2))
+ PQgetisnull(res, 0, 1))
{
pg_fatal("unexpected NULL result while fetching remote files\n");
}
@@ -278,6 +283,20 @@ receiveFileChunks(const char *sql)
memcpy(filename, PQgetvalue(res, 0, 0), filenamelen);
filename[filenamelen] = '\0';
+ /*
+ * It may be possible that a file has been deleted on remote side after
+ * creating the file map. In this case simply ignore it and move on.
+ */
+ if (PQgetisnull(res, 0, 2))
+ {
+ pg_log(PG_DEBUG,
+ "received NULL chunk for file \"%s\", file has been deleted\n",
+ filename);
+ pg_free(filename);
+ PQclear(res);
+ continue;
+ }
+
chunk = PQgetvalue(res, 0, 2);
pg_log(PG_DEBUG, "received chunk for file \"%s\", off %d, len %d\n",
@@ -445,7 +464,7 @@ libpq_executeFileMap(filemap_t *map)
*/
sql =
"SELECT path, begin, \n"
- " pg_read_binary_file(path, begin, len) AS chunk\n"
+ " pg_read_binary_file(path, begin, len, true) AS chunk\n"
"FROM fetchchunks\n";
receiveFileChunks(sql);
--
2.4.4
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index 7e54ac5..ea5385c 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -291,6 +291,12 @@ main(int argc, char **argv) print_filemap(); /* + * Sleep here to allow user to do nasty things with the source file + * list. + */ + pg_usleep(10000000); /* 10s */ + + /* * Ok, we're ready to start copying things over. */ if (showprogress)
rewind_test.bash
Description: Binary data
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers
