I wrote:
> On 2/8/19, Tom Lane <[email protected]> wrote:
>> A script such as you suggest might be a good way to reduce the temptation
>> to get lazy at the last minute. Now that the catalog data is pretty
>> machine-readable, I suspect it wouldn't be very hard --- though I'm
>> not volunteering either. I'm envisioning something simple like "renumber
>> all OIDs in range mmmm-nnnn into range xxxx-yyyy", perhaps with the
>> ability to skip any already-used OIDs in the target range.
>
> This might be something that can be done inside reformat_dat_files.pl.
> It's a little outside it's scope, but better than the alternatives.
Along those lines, here's a draft patch to do just that. It handles
array type oids as well. Run it like this:
perl reformat_dat_file.pl --map-from 9000 --map-to 2000 *.dat
There is some attempt at documentation. So far it doesn't map by
default, but that could be changed if we agreed on the convention of
9000 or whatever.
--
John Naylor https://www.2ndQuadrant.com/
PostgreSQL Development, 24x7 Support, Remote DBA, Training & Services
diff --git a/doc/src/sgml/bki.sgml b/doc/src/sgml/bki.sgml
index 3c5830bc8a..27b9b52e33 100644
--- a/doc/src/sgml/bki.sgml
+++ b/doc/src/sgml/bki.sgml
@@ -392,6 +392,22 @@
at compile time.)
</para>
+ <para>
+ For a variety of reasons, newly assigned OIDs should occupy the lower
+ end of the reserved range. Because multiple patches considered for
+ inclusion into <productname>PostgreSQL</productname> could be using
+ the same new OIDs, there is the possibilty of conflict. To mitigate
+ this, there is a convention that OIDs 9000 and over are reserved for
+ development. This reduces the effort in rebasing over the HEAD branch.
+ To enable committers to move these OIDs to the lower range easily,
+ <filename>reformat_dat_file.pl</filename> can map OIDs as in the
+ following, which maps any OIDs 9000 and over to unused OIDs starting
+ at 2000:
+<programlisting>
+$ perl reformat_dat_file.pl --map-from 9000 --map-to 2000 *.dat
+</programlisting>
+ </para>
+
<para>
The OID counter starts at 10000 at the beginning of a bootstrap run.
If a row from a source other than <filename>postgres.bki</filename>
diff --git a/src/include/catalog/Makefile b/src/include/catalog/Makefile
index ae797d2465..33fbcf7677 100644
--- a/src/include/catalog/Makefile
+++ b/src/include/catalog/Makefile
@@ -13,19 +13,16 @@ subdir = src/include/catalog
top_builddir = ../../..
include $(top_builddir)/src/Makefile.global
-# location of Catalog.pm
-catalogdir = $(top_srcdir)/src/backend/catalog
-
# 'make reformat-dat-files' is a convenience target for rewriting the
# catalog data files in our standard format. This includes collapsing
# out any entries that are redundant with a BKI_DEFAULT annotation.
reformat-dat-files:
- $(PERL) -I $(catalogdir) $(srcdir)/reformat_dat_file.pl -o $(srcdir) $(srcdir)/pg_*.dat
+ $(PERL) $(srcdir)/reformat_dat_file.pl --output $(srcdir) $(srcdir)/pg_*.dat
# 'make expand-dat-files' is a convenience target for expanding out all
# default values in the catalog data files. This should be run before
# altering or removing any BKI_DEFAULT annotation.
expand-dat-files:
- $(PERL) -I $(catalogdir) $(srcdir)/reformat_dat_file.pl -o $(srcdir) $(srcdir)/pg_*.dat --full-tuples
+ $(PERL) $(srcdir)/reformat_dat_file.pl --output $(srcdir) $(srcdir)/pg_*.dat --full-tuples
.PHONY: reformat-dat-files expand-dat-files
diff --git a/src/include/catalog/reformat_dat_file.pl b/src/include/catalog/reformat_dat_file.pl
index 4835c2e41b..05c157ed48 100755
--- a/src/include/catalog/reformat_dat_file.pl
+++ b/src/include/catalog/reformat_dat_file.pl
@@ -19,10 +19,14 @@
use strict;
use warnings;
+use Getopt::Long;
+
+# Must run in src/include/catalog
+use FindBin;
+chdir $FindBin::RealBin or die "could not cd to $FindBin::RealBin: $!\n";
# If you copy this script to somewhere other than src/include/catalog,
# you'll need to modify this "use lib" or provide a suitable -I switch.
-use FindBin;
use lib "$FindBin::RealBin/../../backend/catalog/";
use Catalog;
@@ -34,35 +38,27 @@ use Catalog;
my @METADATA =
('oid', 'oid_symbol', 'array_type_oid', 'descr', 'autogenerated');
-my @input_files;
+my $FirstGenbkiObjectId =
+ Catalog::FindDefinedSymbol('access/transam.h', '..',
+ 'FirstGenbkiObjectId');
+
my $output_path = '';
my $full_tuples = 0;
-# Process command line switches.
-while (@ARGV)
-{
- my $arg = shift @ARGV;
- if ($arg !~ /^-/)
- {
- push @input_files, $arg;
- }
- elsif ($arg =~ /^-o/)
- {
- $output_path = length($arg) > 2 ? substr($arg, 2) : shift @ARGV;
- }
- elsif ($arg eq '--full-tuples')
- {
- $full_tuples = 1;
- }
- else
- {
- usage();
- }
-}
+# Don't map any OIDs by default
+my $min_reserved_oid = $FirstGenbkiObjectId;
+my $first_target_oid = 1;
+
+GetOptions(
+ 'output:s' => \$output_path,
+ 'full-tuples' => \$full_tuples,
+ 'map-from:i' => \$min_reserved_oid,
+ 'map-to:i' => \$first_target_oid) || usage();
# Sanity check arguments.
-die "No input files.\n"
- if !@input_files;
+die "No input files.\n" unless @ARGV;
+die "map-to is higher than map-from"
+ if $first_target_oid > $min_reserved_oid;
# Make sure output_path ends in a slash.
if ($output_path ne '' && substr($output_path, -1) ne '/')
@@ -76,7 +72,7 @@ if ($output_path ne '' && substr($output_path, -1) ne '/')
my %catalogs;
my %catalog_data;
my @catnames;
-foreach my $datfile (@input_files)
+foreach my $datfile (@ARGV)
{
$datfile =~ /(.+)\.dat$/
or die "Input files need to be data (.dat) files.\n";
@@ -95,6 +91,13 @@ foreach my $datfile (@input_files)
$catalog_data{$catname} = Catalog::ParseData($datfile, $schema, 1);
}
+# Collect all the assigned OIDs in numerical order.
+my @header_files = (glob("pg_*.h"), qw(indexing.h toasting.h));
+my $oids = [];
+$oids = Catalog::FindAllOidsFromHeaders(@header_files)
+ if $min_reserved_oid < $FirstGenbkiObjectId;
+my @oids = sort { $a <=> $b } @$oids;
+
########################################################################
# At this point, we have read all the data. If you are modifying this
# script for bulk editing, this is a good place to build lookup tables,
@@ -113,6 +116,7 @@ foreach my $datfile (@input_files)
########################################################################
# Write the data.
+my $next_oid = $first_target_oid;
foreach my $catname (@catnames)
{
my $catalog = $catalogs{$catname};
@@ -146,8 +150,8 @@ foreach my $catname (@catnames)
############################################################
# At this point we have the full tuple in memory as a hash
# and can do any operations we want. As written, it only
- # removes default values, but this script can be adapted to
- # do one-off bulk-editing.
+ # removes redundant info and possibly renumbers OIDs, but
+ # this script can be adapted to do one-off bulk-editing.
############################################################
if (!$full_tuples)
@@ -158,6 +162,14 @@ foreach my $catname (@catnames)
strip_default_values(\%values, $schema, $catname);
}
+ $values{oid} = get_lower_oid()
+ if defined $values{oid}
+ && $values{oid} >= $min_reserved_oid;
+
+ $values{array_type_oid} = get_lower_oid()
+ if defined $values{array_type_oid}
+ && $values{array_type_oid} >= $min_reserved_oid;
+
print $dat "{";
# Separate out metadata fields for readability.
@@ -312,14 +324,49 @@ sub format_hash
return $hash_str;
}
+# Return the lowest OID above $first_target_oid that hasn't been assigned.
+sub get_lower_oid
+{
+ my $new_oid;
+
+ for (;;)
+ {
+ if (scalar @oids == 0 || $next_oid < $oids[0])
+ {
+ $new_oid = $next_oid;
+ warn sprintf
+ "failed to map OID %d out of reserved upper range\n",
+ $new_oid
+ if $new_oid >= $min_reserved_oid;
+ $next_oid++;
+ return $new_oid;
+ }
+ elsif ($next_oid > $oids[0])
+ {
+ shift @oids;
+ }
+ elsif ($oids[0] == $next_oid)
+ {
+ shift @oids;
+ $next_oid++;
+ }
+ else
+ {
+ die "unreachable\n";
+ }
+ }
+}
+
sub usage
{
die <<EOM;
-Usage: reformat_dat_file.pl [options] datafile...
+Usage: reformat_dat_file.pl [--output <path>] [--map-from X] [--map-to Y] [--full-tuples] datafile...
Options:
- -o PATH write output files to PATH instead of current directory
+ --output output directory (default '.')
--full-tuples write out full tuples, including default values
+ --map-from OIDs greater or equal to this will be mapped to lower ones (default none)
+ --map-to first OID to map to
Expects a list of .dat files as arguments.
