Author: spadkins
Date: Fri Jan 6 08:44:44 2012
New Revision: 15070
Modified:
p5ee/trunk/App-Repository/CHANGES
p5ee/trunk/App-Repository/lib/App/Repository.pm
Log:
support metadata caching on disk. (first release candidate)
Modified: p5ee/trunk/App-Repository/CHANGES
==============================================================================
--- p5ee/trunk/App-Repository/CHANGES (original)
+++ p5ee/trunk/App-Repository/CHANGES Fri Jan 6 08:44:44 2012
@@ -2,6 +2,17 @@
# CHANGE LOG
#########################################
+0.969 PRERELEASE
+ x added PrintWarn => 0 to the dbh attributes (useful for Oracle
execute_array() to suppress primary key violation warnings)
+ x support db->_do() for select statements which start with whitespace
+ x actually specify what Oracle errors a reconnect is allowed on
+ x support arbitrary DBI connection attributes to be set. (Useful for Oracle
DRCP, etc.) (e.g. {repname}.dbiattr = xyz,foo=5)
+ x support repository and table metadata caching on disk (HUGE PERFORMANCE
IMPROVEMENT)
+ Prior to this, App-Repository was built for ease-of-programming. Every time
it would connect, it would query the database
+ for a list of the tables and for the list and definition of types. Every
time it accessed a table, it would query the database
+ for the columns (and possibly also the keys/unique indexes). Now, using the
dbmetadata tool, you can extract this metadata
+ from the database and save a lot of overhead.
+
0.968
x summary_tables are now implemented
x queries which include undefined columns in the order by clause exclude
those columns and work correctly otherwise
Modified: p5ee/trunk/App-Repository/lib/App/Repository.pm
==============================================================================
--- p5ee/trunk/App-Repository/lib/App/Repository.pm (original)
+++ p5ee/trunk/App-Repository/lib/App/Repository.pm Fri Jan 6 08:44:44 2012
@@ -4228,10 +4228,32 @@
&App::sub_entry if ($App::trace);
my ($self) = @_;
- my ($table, $tables, $table_defs, $table_def, $native_table, $idx, $label,
@label);
+ my ($table, $tables, $rep_def, $table_defs, $table_def, $native_table,
$idx, $label, @label);
- # load up all possible information from the native metadata
- $self->_load_rep_metadata_from_source();
+ my $context = $self->{context};
+ my $options = $context->{options};
+ my $prefix = $options->{prefix};
+ my $rep_file = "$prefix/etc/app/Repository/$self->{name}/_META_.pl";
+ #print $self->dump();
+ #print STDERR "_load_rep_metadata(): checking $rep_file\n";
+
+ if (! $options->{force_metadata_load} && -r $rep_file) {
+ #print STDERR "_load_rep_metadata($table): checking $rep_file : found
AND {force_metadata_load}=[$options->{force_metadata_load}] is false\n";
+ $rep_def = do $rep_file;
+ if (!$rep_def) {
+ die "Error: Syntax error in $rep_file: $@" if ($@);
+ die "Error: Couldn't do $rep_file: $!" if (! defined $rep_def);
+ die "Error: Couldn't run $rep_file" if (! $rep_def);
+ }
+ App::Reference->overlay($self, $rep_def);
+ }
+ else {
+ #print STDERR "_load_rep_metadata($table): checking $rep_file : not
found OR {force_metadata_load}=[$options->{force_metadata_load}] is true\n";
+ $self->_load_rep_metadata_from_source();
+ if ($options->{force_metadata_write}) {
+ $self->_write_rep_metadata("$rep_file.new");
+ }
+ }
# start with the list of tables that was configured (or the empty list)
$tables = $self->{tables};
@@ -4318,6 +4340,32 @@
&App::sub_exit() if ($App::trace);
}
+sub _write_rep_metadata {
+ &App::sub_entry if ($App::trace);
+ my ($self, $rep_file) = @_;
+
+ open(my $fh, ">", $rep_file) || die "Can't open $rep_file for writing: $!";
+
+ my $name = $self->{name};
+ my $dump_name = "Repository__${name}";
+ my $copy = { %$self };
+ my ($key);
+ foreach $key qw(context _repository name class) {
+ delete $copy->{$key};
+ }
+ foreach $key (keys %$copy) {
+ delete $copy->{$key} if ($key =~ /^db/ || $key =~ /^$name.db/);
+ }
+
+ my $d = Data::Dumper->new([ $copy ], [ $dump_name ]);
+ $d->Indent(1);
+ $d->Purity(1);
+ print $fh $d->Dump();
+
+ close($fh);
+ &App::sub_exit() if ($App::trace);
+}
+
#############################################################################
# _load_rep_metadata_from_source()
#############################################################################
@@ -4395,25 +4443,36 @@
$table_def = $self->{table}{$table} || {};
#print STDERR "_load_table_metadata($table): table_def=[$table_def] ",
($table_def ? "{".join("|", %$table_def)."}" : "undef"), "\n";
- if (!$table_def->{column}) { # no columns are defined
- my $context = $self->{context};
- my $options = $context->{options};
- my $prefix = $options->{prefix};
- my $conf_type = $options->{conf_type} || "pl";
- my $table_file =
"$prefix/etc/app/Repository/$self->{name}/$table.$conf_type";
- #print STDERR "_load_table_metadata($table): checking $table_file\n";
- if (-r $table_file) {
- #print STDERR "_load_table_metadata($table): checking $table_file
: found\n";
- $table_def = do $table_file;
- if ($table_def->{overlay}) {
- delete $table_def->{overlay};
- # Caution. Overlays the entire conf, not just the table_def.
Use with care.
- App::Reference->overlay($self->{context}{conf}, $table_def);
- }
- else {
- # Normal. Replaces the table_def.
- $self->{table}{$table} = $table_def;
- }
+ #print STDERR $self->dump($table_def);
+
+ my $context = $self->{context};
+ my $options = $context->{options};
+ my $prefix = $options->{prefix};
+ my $table_file = "$prefix/etc/app/Repository/$self->{name}/$table.pl";
+ #print STDERR "_load_table_metadata($table): checking $table_file\n";
+ if (! $options->{force_metadata_load} && -r $table_file) {
+ #print STDERR "_load_table_metadata($table): checking $table_file :
found\n";
+ $table_def = do $table_file;
+ if (!$table_def) {
+ die "Error: Syntax error in $table_file: $@" if ($@);
+ die "Error: Couldn't do $table_file: $!" if (! defined
$table_def);
+ die "Error: Couldn't run $table_file" if (! $table_def);
+ }
+ if ($table_def->{overlay}) {
+ delete $table_def->{overlay};
+ # Caution. Overlays the entire conf, not just the table_def. Use
with care.
+ App::Reference->overlay($self->{context}{conf}, $table_def);
+ }
+ else {
+ # Normal. Replaces the table_def.
+ $self->{table}{$table} = $table_def;
+ }
+ }
+ else {
+ # load up all additional information from the native metadata
+ $self->_load_table_metadata_from_source($table);
+ if ($options->{force_metadata_write}) {
+ $self->_write_table_metadata("$table_file.new", $table);
}
}
@@ -4424,9 +4483,6 @@
return;
}
- # load up all additional information from the native metadata
- $self->_load_table_metadata_from_source($table);
-
if ($table_def->{overlay_from_table}) {
#print STDERR "load_table_metadata($table) :
OVERLAY=[$table_def->{overlay_from_table}]\n";
@@ -4596,12 +4652,41 @@
}
}
}
-
+
+ # NOTE: This must be only perl code without access to the database.
Otherwise the metadata caching doesn't work.
$self->_load_table_metadata_from_source2($table);
&App::sub_exit() if ($App::trace);
}
+sub _write_table_metadata {
+ &App::sub_entry if ($App::trace);
+ my ($self, $table_file, $table) = @_;
+
+ open(my $fh, ">", $table_file) || die "Can't open $table_file for writing:
$!";
+
+ my $name = $self->{name};
+ my $dump_name = "Repository__${name}__${table}";
+ my $table_def = $self->{table}{$table} || {};
+ my $copy = { %$table_def };
+
+ my ($key);
+ #foreach $key qw(context _repository name class) {
+ # delete $copy->{$key};
+ #}
+ #foreach $key (keys %$copy) {
+ # delete $copy->{$key} if ($key =~ /^db/ || $key =~ /^$name.db/);
+ #}
+
+ my $d = Data::Dumper->new([ $copy ], [ $dump_name ]);
+ $d->Indent(1);
+ $d->Purity(1);
+ print $fh $d->Dump();
+
+ close($fh);
+ &App::sub_exit() if ($App::trace);
+}
+
#############################################################################
# _load_table_metadata_from_source()
#############################################################################
@@ -4628,10 +4713,12 @@
=cut
+# NOTE: This generally will access the database. It does not get called if
cached metadata exists.
sub _load_table_metadata_from_source {
my ($self, $table) = @_;
}
+# NOTE: This must be only perl code without access to the database. Otherwise
the metadata caching doesn't work.
sub _load_table_metadata_from_source2 {
my ($self, $table) = @_;
}
