On Thu, Jul 10, 2014 at 06:38:26PM -0400, Bruce Momjian wrote:
> On Thu, Jul 10, 2014 at 06:17:14PM -0400, Bruce Momjian wrote:
> > Well, we are going to need to call internal C functions, often bypassing
> > their typical call sites and the assumption about locking, etc. Perhaps
> > this could be done from a plpgsql function. We could add and drop a
> > dummy column to force TOAST table creation --- we would then only need a
> > way to detect if a function _needs_ a TOAST table, which was skipped in
> > binary upgrade mode previously.
> >
> > That might be a minimalistic approach.
>
> I have thought some more on this. I thought I would need to open
> pg_class in C and do complex backend stuff, but I now realize I can do
> it from libpq, and just call ALTER TABLE and I think that always
> auto-checks if a TOAST table is needed. All I have to do is query
> pg_class from libpq, then construct ALTER TABLE commands for each item,
> and it will optionally create the TOAST table if needed. I just have to
> use a no-op ALTER TABLE command, like SET STATISTICS.
Attached is a completed patch which handles oid conflicts in pg_class
and pg_type for TOAST tables that were not needed in the old cluster but
are needed in the new one. I was able to recreate a failure case and
this fixed it.
The patch need to be backpatched because I am getting more-frequent bug
reports from users using pg_upgrade to leave now-end-of-life'ed 8.4.
There is not a good work-around for pg_upgrade failures without this
fix, but at least pg_upgrade throws an error.
--
Bruce Momjian <[email protected]> http://momjian.us
EnterpriseDB http://enterprisedb.com
+ Everyone has their own god. +
diff --git a/contrib/pg_upgrade/dump.c b/contrib/pg_upgrade/dump.c
new file mode 100644
index b112f3a..642dd5d
*** a/contrib/pg_upgrade/dump.c
--- b/contrib/pg_upgrade/dump.c
***************
*** 12,17 ****
--- 12,19 ----
#include "pg_upgrade.h"
#include <sys/types.h>
+ #include "catalog/binary_upgrade.h"
+
void
generate_old_dump(void)
*************** generate_old_dump(void)
*** 67,69 ****
--- 69,139 ----
end_progress_output();
check_ok();
}
+
+
+ /*
+ * It is possible for there to be a mismatch in the need for TOAST tables
+ * between the old and new servers, e.g. some pre-9.1 tables didn't need
+ * TOAST tables but will need them in 9.1+. (There are also opposite cases,
+ * but these are handled by setting binary_upgrade_next_toast_pg_class_oid.)
+ *
+ * We can't allow the TOAST table to be created by pg_dump with a
+ * pg_dump-assigned oid because it might conflict with a later table that
+ * uses that oid, causing a "file exists" error for pg_class conflicts, and
+ * a "duplicate oid" error for pg_type conflicts. (TOAST tables need pg_type
+ * entries.)
+ *
+ * Therefore, a backend in binary-upgrade mode will not create a TOAST
+ * table unless an OID as passed in via pg_upgrade_support functions.
+ * This function is called after the restore and uses ALTER TABLE to
+ * auto-create any needed TOAST tables which will not conflict with
+ * restored oids.
+ */
+ void
+ optionally_create_toast_tables(void)
+ {
+ int dbnum;
+
+ prep_status("Creating newly-required TOAST tables");
+
+ for (dbnum = 0; dbnum < new_cluster.dbarr.ndbs; dbnum++)
+ {
+ PGresult *res;
+ int ntups;
+ int rowno;
+ int i_nspname,
+ i_relname;
+ DbInfo *active_db = &new_cluster.dbarr.dbs[dbnum];
+ PGconn *conn = connectToServer(&new_cluster, active_db->db_name);
+
+ res = executeQueryOrDie(conn,
+ "SELECT n.nspname, c.relname "
+ "FROM pg_catalog.pg_class c, "
+ " pg_catalog.pg_namespace n "
+ "WHERE c.relnamespace = n.oid AND "
+ " n.nspname NOT IN ('pg_catalog', 'information_schema') AND "
+ "c.relkind IN ('r', 'm') AND "
+ "c.reltoastrelid = 0");
+
+ ntups = PQntuples(res);
+ i_nspname = PQfnumber(res, "nspname");
+ i_relname = PQfnumber(res, "relname");
+ for (rowno = 0; rowno < ntups; rowno++)
+ {
+ /* enable auto-oid-numbered TOAST creation if needed */
+ PQclear(executeQueryOrDie(conn, "SELECT binary_upgrade.set_next_toast_pg_class_oid('%d'::pg_catalog.oid);",
+ OPTIONALLY_CREATE_TOAST_OID));
+
+ /* dummy command that also triggers check for required TOAST table */
+ PQclear(executeQueryOrDie(conn, "ALTER TABLE %s.%s RESET (binary_upgrade_dummy_option);",
+ quote_identifier(PQgetvalue(res, rowno, i_nspname)),
+ quote_identifier(PQgetvalue(res, rowno, i_relname))));
+ }
+
+ PQclear(res);
+
+ PQfinish(conn);
+ }
+
+ check_ok();
+ }
diff --git a/contrib/pg_upgrade/pg_upgrade.c b/contrib/pg_upgrade/pg_upgrade.c
new file mode 100644
index 3e97b66..49ea873
*** a/contrib/pg_upgrade/pg_upgrade.c
--- b/contrib/pg_upgrade/pg_upgrade.c
*************** create_new_objects(void)
*** 363,368 ****
--- 363,370 ----
if (GET_MAJOR_VERSION(old_cluster.major_version) < 903)
set_frozenxids(true);
+ optionally_create_toast_tables();
+
/* regenerate now that we have objects in the databases */
get_db_and_rel_infos(&new_cluster);
diff --git a/contrib/pg_upgrade/pg_upgrade.h b/contrib/pg_upgrade/pg_upgrade.h
new file mode 100644
index 61e5de0..1175604
*** a/contrib/pg_upgrade/pg_upgrade.h
--- b/contrib/pg_upgrade/pg_upgrade.h
*************** void disable_old_cluster(void);
*** 336,341 ****
--- 336,342 ----
/* dump.c */
void generate_old_dump(void);
+ void optionally_create_toast_tables(void);
/* exec.c */
diff --git a/src/backend/catalog/toasting.c b/src/backend/catalog/toasting.c
new file mode 100644
index bdfeb90..e2549b2
*** a/src/backend/catalog/toasting.c
--- b/src/backend/catalog/toasting.c
*************** create_toast_table(Relation rel, Oid toa
*** 165,180 ****
if (rel->rd_rel->reltoastrelid != InvalidOid)
return false;
! /*
! * Check to see whether the table actually needs a TOAST table.
! *
! * If an update-in-place toast relfilenode is specified, force toast file
! * creation even if it seems not to need one.
! */
! if (!needs_toast_table(rel) &&
! (!IsBinaryUpgrade ||
! !OidIsValid(binary_upgrade_next_toast_pg_class_oid)))
! return false;
/*
* If requested check lockmode is sufficient. This is a cross check in
--- 165,211 ----
if (rel->rd_rel->reltoastrelid != InvalidOid)
return false;
! if (!IsBinaryUpgrade)
! {
! if (!needs_toast_table(rel))
! return false;
! }
! else
! {
! /*
! * Check to see whether the table needs a TOAST table.
! *
! * If an update-in-place TOAST relfilenode is specified, force TOAST file
! * creation even if it seems not to need one. This handles the case
! * where the old cluster needed a TOAST table but the new cluster
! * would not normally create one.
! */
!
! /*
! * If a TOAST oid is not specified, skip TOAST creation as we will do
! * it later so we don't create a TOAST table whose OID later conflicts
! * with a user-supplied OID. This handles cases where the old cluster
! * didn't need a TOAST table, but the new cluster does.
! */
! if (!OidIsValid(binary_upgrade_next_toast_pg_class_oid))
! return false;
!
! /*
! * If a special TOAST value has been passed in, it means we are in
! * cleanup mode --- we are creating needed TOAST tables after all user
! * tables with specified OIDs have been created. We let the system
! * assign a TOAST oid for us. The tables are empty so the missing
! * TOAST tables were not a problem.
! */
! if (binary_upgrade_next_toast_pg_class_oid == OPTIONALLY_CREATE_TOAST_OID)
! {
! /* clear as it is not to be used; it is just a flag */
! binary_upgrade_next_toast_pg_class_oid = InvalidOid;
!
! if (!needs_toast_table(rel))
! return false;
! }
! }
/*
* If requested check lockmode is sufficient. This is a cross check in
diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h
new file mode 100644
index f39017c..63fa85e
*** a/src/include/catalog/binary_upgrade.h
--- b/src/include/catalog/binary_upgrade.h
***************
*** 14,19 ****
--- 14,24 ----
#ifndef BINARY_UPGRADE_H
#define BINARY_UPGRADE_H
+ #include "catalog/pg_authid.h"
+
+ /* pick a OID that will never be used for TOAST tables */
+ #define OPTIONALLY_CREATE_TOAST_OID BOOTSTRAP_SUPERUSERID
+
extern PGDLLIMPORT Oid binary_upgrade_next_pg_type_oid;
extern PGDLLIMPORT Oid binary_upgrade_next_array_pg_type_oid;
extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_type_oid;
--
Sent via pgsql-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
