On Thu, 25 May 2023 at 23:04, Dagfinn Ilmari Mannsåker
<ilm...@ilmari.org> wrote:
>
> vignesh C <vignes...@gmail.com> writes:
>
> > Hi,
> >
> > The recovery tap test has 4 implementations of find_in_log sub routine
> > for various uses, I felt we can generalize these and have a single
> > function for the same. The attached patch is an attempt to have a
> > generalized sub routine find_in_log which can be used by all of them.
> > Thoughts?
>
> +1 on factoring out this common code. Just a few comments on the
> implementation.
>
>
> > diff --git a/src/test/perl/PostgreSQL/Test/Utils.pm
> > b/src/test/perl/PostgreSQL/Test/Utils.pm
> > index a27fac83d2..5c9b2f6c03 100644
> > --- a/src/test/perl/PostgreSQL/Test/Utils.pm
> > +++ b/src/test/perl/PostgreSQL/Test/Utils.pm
> > @@ -67,6 +67,7 @@ our @EXPORT = qw(
> > slurp_file
> > append_to_file
> > string_replace_file
> > + find_in_log
> > check_mode_recursive
> > chmod_recursive
> > check_pg_config
> > @@ -579,6 +580,28 @@ sub string_replace_file
> >
> > =pod
> >
> > +
> > +=item find_in_log(node, pattern, offset)
> > +
> > +Find pattern in logfile of node after offset byte.
> > +
> > +=cut
> > +
> > +sub find_in_log
> > +{
> > + my ($node, $pattern, $offset) = @_;
> > +
> > + $offset = 0 unless defined $offset;
> > + my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
>
> Since this function is in the same package, there's no need to qualify
> it with the full name. I know the callers you copied it from did, but
> they wouldn't have had to either, since it's exported by default (in the
> @EXPORT array above), unless the use statement has an explicit argument
> list that excludes it.
I have moved this function to Cluster.pm file now, since it is moved,
I had to qualify the name with the full name.
> > + return 0 if (length($log) <= 0 || length($log) <= $offset);
> > +
> > + $log = substr($log, $offset);
>
> Also, the existing callers don't seem to have got the memo that
> slurp_file() takes an optinal offset parameter, which will cause it to
> seek to that postion before slurping the file, which is more efficient
> than reading the whole file in and substr-ing it. There's not much
> point in the length checks either, since regex-matching against an empty
> string is very cheap (and if the provide pattern can match the empty
> string the whole function call is rather pointless).
>
> > + return $log =~ m/$pattern/;
> > +}
>
> All in all, it could be simplified to:
>
> sub find_in_log {
> my ($node, $pattern, $offset) = @_;
>
> return slurp_file($node->logfile, $offset) =~ $pattern;
> }
Modified in similar lines
> However, none of the other functions in ::Utils know anything about node
> objects, which makes me think it should be a method on the node itself
> (i.e. in PostgreSQL::Test::Cluster) instead. Also, I think log_contains
> would be a better name, since it just returns a boolean. The name
> find_in_log makes me think it would return the log lines matching the
> pattern, or the position of the match in the file.
Modified
> In that case, the slurp_file() call would have to be fully qualified,
> since ::Cluster uses an empty import list to avoid polluting the method
> namespace with imported functions.
Modified.
Thanks for the comments, the attached v2 version patch has the changes
for the same.
Regards,
Vignesh
From 8df06d233de4e5b5abe7a4dd4bc314c215b2dfc2 Mon Sep 17 00:00:00 2001
From: Vignesh C <vignes...@gmail.com>
Date: Fri, 26 May 2023 20:07:30 +0530
Subject: [PATCH v2] Remove duplicate find_in_log sub routines from tap tests.
Remove duplicate find_in_log sub routines from tap tests.
---
src/test/authentication/t/003_peer.pl | 17 ++--------
src/test/perl/PostgreSQL/Test/Cluster.pm | 16 +++++++++
src/test/recovery/t/019_replslot_limit.pl | 33 +++++--------------
src/test/recovery/t/033_replay_tsp_drops.pl | 15 ++-------
.../t/035_standby_logical_decoding.pl | 26 +++------------
5 files changed, 33 insertions(+), 74 deletions(-)
diff --git a/src/test/authentication/t/003_peer.pl b/src/test/authentication/t/003_peer.pl
index 3272e52cae..2a035c2d0d 100644
--- a/src/test/authentication/t/003_peer.pl
+++ b/src/test/authentication/t/003_peer.pl
@@ -69,17 +69,6 @@ sub test_role
}
}
-# Find $pattern in log file of $node.
-sub find_in_log
-{
- my ($node, $offset, $pattern) = @_;
-
- my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile, $offset);
- return 0 if (length($log) <= 0);
-
- return $log =~ m/$pattern/;
-}
-
my $node = PostgreSQL::Test::Cluster->new('node');
$node->init;
$node->append_conf('postgresql.conf', "log_connections = on\n");
@@ -91,9 +80,9 @@ reset_pg_hba($node, 'peer');
# Check if peer authentication is supported on this platform.
my $log_offset = -s $node->logfile;
$node->psql('postgres');
-if (find_in_log(
- $node, $log_offset,
- qr/peer authentication is not supported on this platform/))
+if ($node->log_contains(
+ qr/peer authentication is not supported on this platform/),
+ $log_offset,)
{
plan skip_all => 'peer authentication is not supported on this platform';
}
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index baea0fcd1c..f10e44ba51 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -2536,6 +2536,22 @@ sub log_content
}
+=pod
+
+=item log_contains(pattern, offset)
+
+Find pattern in logfile of node after offset byte.
+
+=cut
+
+sub log_contains
+{
+ my ($self, $pattern, $offset) = @_;
+
+ $offset = 0 unless defined $offset;
+ return PostgreSQL::Test::Utils::slurp_file($self->logfile, $offset) =~ m/$pattern/;
+}
+
=pod
=item $node->run_log(...)
diff --git a/src/test/recovery/t/019_replslot_limit.pl b/src/test/recovery/t/019_replslot_limit.pl
index a1aba16e14..95acf9e357 100644
--- a/src/test/recovery/t/019_replslot_limit.pl
+++ b/src/test/recovery/t/019_replslot_limit.pl
@@ -161,8 +161,7 @@ $node_primary->wait_for_catchup($node_standby);
$node_standby->stop;
-ok( !find_in_log(
- $node_standby,
+ok( !$node_standby->log_contains(
"requested WAL segment [0-9A-F]+ has already been removed"),
'check that required WAL segments are still available');
@@ -184,8 +183,8 @@ $node_primary->safe_psql('postgres', "CHECKPOINT;");
my $invalidated = 0;
for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++)
{
- if (find_in_log(
- $node_primary, 'invalidating obsolete replication slot "rep1"',
+ if ($node_primary->log_contains(
+ 'invalidating obsolete replication slot "rep1"',
$logstart))
{
$invalidated = 1;
@@ -207,7 +206,7 @@ is($result, "rep1|f|t|lost|",
my $checkpoint_ended = 0;
for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++)
{
- if (find_in_log($node_primary, "checkpoint complete: ", $logstart))
+ if ($node_primary->log_contains("checkpoint complete: ", $logstart))
{
$checkpoint_ended = 1;
last;
@@ -237,8 +236,7 @@ $node_standby->start;
my $failed = 0;
for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++)
{
- if (find_in_log(
- $node_standby,
+ if ($node_standby->log_contains(
"requested WAL segment [0-9A-F]+ has already been removed",
$logstart))
{
@@ -381,8 +379,7 @@ my $msg_logged = 0;
my $max_attempts = $PostgreSQL::Test::Utils::timeout_default;
while ($max_attempts-- >= 0)
{
- if (find_in_log(
- $node_primary3,
+ if ($node_primary3->log_contains(
"terminating process $senderpid to release replication slot \"rep3\"",
$logstart))
{
@@ -406,8 +403,8 @@ $msg_logged = 0;
$max_attempts = $PostgreSQL::Test::Utils::timeout_default;
while ($max_attempts-- >= 0)
{
- if (find_in_log(
- $node_primary3, 'invalidating obsolete replication slot "rep3"',
+ if ($node_primary3->log_contains(
+ 'invalidating obsolete replication slot "rep3"',
$logstart))
{
$msg_logged = 1;
@@ -446,18 +443,4 @@ sub get_log_size
return (stat $node->logfile)[7];
}
-# find $pat in logfile of $node after $off-th byte
-sub find_in_log
-{
- my ($node, $pat, $off) = @_;
-
- $off = 0 unless defined $off;
- my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
- return 0 if (length($log) <= $off);
-
- $log = substr($log, $off);
-
- return $log =~ m/$pat/;
-}
-
done_testing();
diff --git a/src/test/recovery/t/033_replay_tsp_drops.pl b/src/test/recovery/t/033_replay_tsp_drops.pl
index 0a35a7bda6..307c30bc6b 100644
--- a/src/test/recovery/t/033_replay_tsp_drops.pl
+++ b/src/test/recovery/t/033_replay_tsp_drops.pl
@@ -135,22 +135,11 @@ while ($max_attempts-- >= 0)
{
last
if (
- find_in_log(
- $node_standby,
+ $node_standby->log_contains(
qr!WARNING: ( [A-Z0-9]+:)? creating missing directory: pg_tblspc/!,
$logstart));
usleep(100_000);
}
ok($max_attempts > 0, "invalid directory creation is detected");
-done_testing();
-
-# find $pat in logfile of $node after $off-th byte
-sub find_in_log
-{
- my ($node, $pat, $off) = @_;
-
- my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile, $off);
-
- return $log =~ m/$pat/;
-}
+done_testing();
\ No newline at end of file
diff --git a/src/test/recovery/t/035_standby_logical_decoding.pl b/src/test/recovery/t/035_standby_logical_decoding.pl
index 64beec4bd3..480e6d6caa 100644
--- a/src/test/recovery/t/035_standby_logical_decoding.pl
+++ b/src/test/recovery/t/035_standby_logical_decoding.pl
@@ -28,20 +28,6 @@ my $res;
my $primary_slotname = 'primary_physical';
my $standby_physical_slotname = 'standby_physical';
-# find $pat in logfile of $node after $off-th byte
-sub find_in_log
-{
- my ($node, $pat, $off) = @_;
-
- $off = 0 unless defined $off;
- my $log = PostgreSQL::Test::Utils::slurp_file($node->logfile);
- return 0 if (length($log) <= $off);
-
- $log = substr($log, $off);
-
- return $log =~ m/$pat/;
-}
-
# Fetch xmin columns from slot's pg_replication_slots row, after waiting for
# given boolean condition to be true to ensure we've reached a quiescent state.
sub wait_for_xmins
@@ -235,14 +221,12 @@ sub check_for_invalidation
my $inactive_slot = $slot_prefix . 'inactiveslot';
# message should be issued
- ok( find_in_log(
- $node_standby,
+ ok( $node_standby->log_contains(
"invalidating obsolete replication slot \"$inactive_slot\"",
$log_start),
"inactiveslot slot invalidation is logged $test_name");
- ok( find_in_log(
- $node_standby,
+ ok( $node_standby->log_contains(
"invalidating obsolete replication slot \"$active_slot\"",
$log_start),
"activeslot slot invalidation is logged $test_name");
@@ -657,14 +641,12 @@ $node_primary->safe_psql(
$node_primary->wait_for_replay_catchup($node_standby);
# message should not be issued
-ok( !find_in_log(
- $node_standby,
+ok( !$node_standby->log_contains(
"invalidating obsolete slot \"no_conflict_inactiveslot\"", $logstart),
'inactiveslot slot invalidation is not logged with vacuum on conflict_test'
);
-ok( !find_in_log(
- $node_standby,
+ok( !$node_standby->log_contains(
"invalidating obsolete slot \"no_conflict_activeslot\"", $logstart),
'activeslot slot invalidation is not logged with vacuum on conflict_test'
);
--
2.34.1
