On Fri, 03 Jan 2003 15:37:56 -0500, Tom Lane <[EMAIL PROTECTED]> wrote: >The system tables are not the problem. [...] > >Changes in the on-disk representation of user tables would be harder to >deal with, but they are also much rarer (AFAIR we've only done that >twice: WAL required additions to page and tuple headers, and then there >were Manfred's space-saving changes in 7.3).
So I'm the bad guy? ;-) AFAICS handling the page and tuple format changes doesn't need much more than what I have hacked together yesterday afternoon: #include <access/htup.h> typedef struct HeapTupleHeader72Data { Oid t_oid; /* OID of this tuple -- 4 bytes */ CommandId t_cmin; /* insert CID stamp -- 4 bytes each */ CommandId t_cmax; /* delete CommandId stamp */ TransactionId t_xmin; /* insert XID stamp -- 4 bytes each */ TransactionId t_xmax; /* delete XID stamp */ ItemPointerData t_ctid; /* current TID of this or newer tuple */ int16 t_natts; /* number of attributes */ uint16 t_infomask; /* various infos */ uint8 t_hoff; /* sizeof() tuple header */ /* ^ - 31 bytes - ^ */ bits8 t_bits[1]; /* bit map of NULLs */ } HeapTupleHeader72Data; typedef HeapTupleHeader72Data *HeapTupleHeader72; /* ** Convert a pre-7.3 heap tuple header to 7.3 format. ** ** On entry ht points to a heap tuple header in 7.2 format, ** which will be converted to the new format in place. ** If compact is true, the size of the heap tuple header ** (t_hoff) is reduced, otherwise enough padding bytes are ** inserted to keep the old length. ** ** The return value is the new size. */ Size HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact) { HeapTupleHeaderData newdata; Oid oid; HeapTupleHeader72 ht72; int len; ht72 = (HeapTupleHeader72) ht; oid = ht72->t_oid; MemSet(&newdata, 0, sizeof(HeapTupleHeaderData)); /* copy fixed fields */ ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid); newdata.t_natts = ht72->t_natts; newdata.t_infomask = ht72->t_infomask; HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin); if (newdata.t_infomask & HEAP_XMAX_INVALID) { HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin); }/*if*/ else { HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax); }/*else*/ if (newdata.t_infomask & HEAP_MOVED) { HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin); }/*if*/ else { HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax); }/*else*/ /* move new structure into original position */ len = offsetof(HeapTupleHeaderData, t_bits); memcpy(ht, &newdata, len); /* copy bitmap (if there is one) */ if (ht->t_infomask & HEAP_HASNULL) { int bitmaplen = BITMAPLEN(ht->t_natts); int off = offsetof(HeapTupleHeader72Data, t_bits); char *p = (char *) ht; int i; Assert(len < off); for (i = 0; i < bitmaplen; ++i) { p[len + i] = p[off + i]; }/*for*/ len += bitmaplen; }/*if*/ /* pad rest with 0 */ Assert(len < ht->t_hoff); memset((char *)ht + len, 0, ht->t_hoff - len); /* change length, if requested */ if (compact) { if (oid != 0) { len += sizeof(Oid); }/*if*/ ht->t_hoff = MAXALIGN(len); }/*if*/ /* copy oid (if there is one) */ if (oid != 0) { ht->t_infomask |= HEAP_HASOID; HeapTupleHeaderSetOid(ht, oid); }/*if*/ return ht->t_hoff; } #include <storage/bufpage.h> #include <access/htup.h> /* ** Convert a pre 7.3 heap page to 7.3 format, ** or leave the page alone, if it is already in 7.3 format. ** ** The page is converted in place. ** ** We should have exclusive access to the page, either per ** LockBufferForCleanup() or because we a running in a standalone ** tool. */ void HeapPage_To73Format(Page page, bool compact) { PageHeader phdr = (PageHeader)page; int version = PageGetPageLayoutVersion(page); Size size = PageGetPageSize(page); int maxoff = PageGetMaxOffsetNumber(page); int i; if (version == PG_PAGE_LAYOUT_VERSION) { /* already converted */ return; }/*if*/ Assert(version == 0); for (i = 1; i <= maxoff; ++i) { ItemId itid = PageGetItemId(page, i); // ??? if (ItemIdIsUsed(itid)) ... HeapTupleHeader ht = PageGetItem(page, itid); Size oldsz = ht->t_hoff; Size newsz; newsz = HeapTupleHeader_To73Format(ht, compact); if (newsz < oldsz) { int diff = oldsz - newsz; ItemOffset off = ItemIdGetOffset(itid); char *addr; int lng; int j; /* move tuple header to the right */ addr = (char *)ht; memmove(addr + diff, addr, newsz); itid->lp_off += diff; itid->lp_len -= diff; /* ** Move all tuples that lie to the left of our tuple header. ** (Shamelessly copied from PageIndexTupleDelete()). */ addr = (char *) page + phdr->pd_upper; lng = (int) (off - phdr->pd_upper); if (lng > 0) memmove(addr + diff, addr, lng); memset(addr, 0, diff); /* ** Adjust upper free space boundary pointer, ** lower is not affected. */ phdr->pd_upper += diff; /* Adjust linp entries. */ for (j = 1; j <= maxoff; ++j) { ItemId ii = PageGetItemId(page, j); if (ii->lp_off < off) ii->lp_off += diff; }/*for*/ }/*if*/ else Assert(newsz == oldsz); }/*for*/ PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION); } /* ** Convert a pre 7.3 page to 7.3 format, ** or leave the page alone, if it is already in 7.3 format. ** ** The page is converted in place. ** ** We should have exclusive access to the page, either per ** LockBufferForCleanup() or because we a running in a standalone ** tool. */ void Page_To73Format(Page page) { int version = PageGetPageLayoutVersion(page); Size size = PageGetPageSize(page); if (version == PG_PAGE_LAYOUT_VERSION) { /* already converted */ return; }/*if*/ Assert(version == 0); if (PageGetSpecialSize(page) == 0) { /* ** Heap page. ** XXX Sure? ** XXX Is there a better way to tell? */ HeapPage_To73Format(page, true); }/*if*/ else { /* ** Not a heap page: no format change, just adjust version */ PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION); }/*else*/ } This should handle all format changes I'm aware of: . bitmap length . overlaying fields . optional oid . page format version Am I missing something? Above code is completely untested, I've not even run it through a compiler; please consider it as a basis for discussion. If there is agreement, that we want 7.2 -> 7.3.x pg_upgrade, I'll put more work into it. What's missing is mainly a call to Page_To73Format() somewhere. I can think of . an input-file-to-output-file-converter run by pg_upgrade instead of copying/moving the files . an in-place-converter run by pg_upgrade after copying/moving the files . converting each page during normal operation immediately after it is fetched from disk. #include <access/htup.h> typedef struct HeapTupleHeader72Data { Oid t_oid; /* OID of this tuple -- 4 bytes */ CommandId t_cmin; /* insert CID stamp -- 4 bytes each */ CommandId t_cmax; /* delete CommandId stamp */ TransactionId t_xmin; /* insert XID stamp -- 4 bytes each */ TransactionId t_xmax; /* delete XID stamp */ ItemPointerData t_ctid; /* current TID of this or newer tuple */ int16 t_natts; /* number of attributes */ uint16 t_infomask; /* various infos */ uint8 t_hoff; /* sizeof() tuple header */ /* ^ - 31 bytes - ^ */ bits8 t_bits[1]; /* bit map of NULLs */ } HeapTupleHeader72Data; typedef HeapTupleHeader72Data *HeapTupleHeader72; /* ** Convert a pre-7.3 heap tuple header to 7.3 format. ** ** On entry ht points to a heap tuple header in 7.2 format, ** which will be converted to the new format in place. ** If compact is true, the size of the heap tuple header ** (t_hoff) is reduced, otherwise enough padding bytes are ** inserted to keep the old length. ** ** The return value is the new size. */ Size HeapTupleHeader_To73Format(HeapTupleHeader ht, bool compact) { HeapTupleHeaderData newdata; Oid oid; HeapTupleHeader72 ht72; int len; ht72 = (HeapTupleHeader72) ht; oid = ht72->t_oid; MemSet(&newdata, 0, sizeof(HeapTupleHeaderData)); /* copy fixed fields */ ItemPointerCopy(&ht72->t_ctid, &newdata.t_ctid); newdata.t_natts = ht72->t_natts; newdata.t_infomask = ht72->t_infomask; HeapTupleHeaderSetXmin(&newdata, ht72->t_xmin); if (newdata.t_infomask & HEAP_XMAX_INVALID) { HeapTupleHeaderSetCmin(&newdata, ht72->t_cmin); }/*if*/ else { HeapTupleHeaderSetXmax(&newdata, ht72->t_xmax); }/*else*/ if (newdata.t_infomask & HEAP_MOVED) { HeapTupleHeaderSetXvac(&newdata, ht72->t_cmin); }/*if*/ else { HeapTupleHeaderSetCmax(&newdata, ht72->t_cmax); }/*else*/ /* move new structure into original position */ len = offsetof(HeapTupleHeaderData, t_bits); memcpy(ht, &newdata, len); /* copy bitmap (if there is one) */ if (ht->t_infomask & HEAP_HASNULL) { int bitmaplen = BITMAPLEN(ht->t_natts); int off = offsetof(HeapTupleHeader72Data, t_bits); char *p = (char *) ht; int i; Assert(len < off); for (i = 0; i < bitmaplen; ++i) { p[len + i] = p[off + i]; }/*for*/ len += bitmaplen; }/*if*/ /* pad rest with 0 */ Assert(len < ht->t_hoff); memset((char *)ht + len, 0, ht->t_hoff - len); /* change length, if requested */ if (compact) { if (oid != 0) { len += sizeof(Oid); }/*if*/ ht->t_hoff = MAXALIGN(len); }/*if*/ /* copy oid (if there is one) */ if (oid != 0) { ht->t_infomask |= HEAP_HASOID; HeapTupleHeaderSetOid(ht, oid); }/*if*/ return ht->t_hoff; } #include <storage/bufpage.h> #include <access/htup.h> /* ** Convert a pre 7.3 heap page to 7.3 format, ** or leave the page alone, if it is already in 7.3 format. ** ** The page is converted in place. ** ** We should have exclusive access to the page, either per ** LockBufferForCleanup() or because we a running in a standalone ** tool. */ void HeapPage_To73Format(Page page, bool compact) { PageHeader phdr = (PageHeader)page; int version = PageGetPageLayoutVersion(page); Size size = PageGetPageSize(page); int maxoff = PageGetMaxOffsetNumber(page); int i; if (version == PG_PAGE_LAYOUT_VERSION) { /* already converted */ return; }/*if*/ Assert(version == 0); for (i = 1; i <= maxoff; ++i) { ItemId itid = PageGetItemId(page, i); // ??? if (ItemIdIsUsed(itid)) ... HeapTupleHeader ht = PageGetItem(page, itid); Size oldsz = ht->t_hoff; Size newsz; newsz = HeapTupleHeader_To73Format(ht, compact); if (newsz < oldsz) { int diff = oldsz - newsz; ItemOffset off = ItemIdGetOffset(itid); char *addr; int lng; int j; /* move tuple header to the right */ addr = (char *)ht; memmove(addr + diff, addr, newsz); itid->lp_off += diff; itid->lp_len -= diff; /* ** Move all tuples that lie to the left of our tuple header. ** (Shamelessly copied from PageIndexTupleDelete()). */ addr = (char *) page + phdr->pd_upper; lng = (int) (off - phdr->pd_upper); if (lng > 0) memmove(addr + diff, addr, lng); memset(addr, 0, diff); /* ** Adjust upper free space boundary pointer, ** lower is not affected. */ phdr->pd_upper += diff; /* Adjust linp entries. */ for (j = 1; j <= maxoff; ++j) { ItemId ii = PageGetItemId(page, j); if (ii->lp_off < off) ii->lp_off += diff; }/*for*/ }/*if*/ else Assert(newsz == oldsz); }/*for*/ PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION); } /* ** Convert a pre 7.3 page to 7.3 format, ** or leave the page alone, if it is already in 7.3 format. ** ** The page is converted in place. ** ** We should have exclusive access to the page, either per ** LockBufferForCleanup() or because we a running in a standalone ** tool. */ void Page_To73Format(Page page) { int version = PageGetPageLayoutVersion(page); Size size = PageGetPageSize(page); if (version == PG_PAGE_LAYOUT_VERSION) { /* already converted */ return; }/*if*/ Assert(version == 0); if (PageGetSpecialSize(page) == 0) { /* ** Heap page. ** XXX Sure? ** XXX Is there a better way to tell? */ HeapPage_To73Format(page, true); }/*if*/ else { /* ** Not a heap page: no format change, just adjust version */ PageSetPageSizeAndVersion(page, size, PG_PAGE_LAYOUT_VERSION); }/*else*/ } Servus Manfred ---------------------------(end of broadcast)--------------------------- TIP 5: Have you checked our extensive FAQ? http://www.postgresql.org/users-lounge/docs/faq.html