From cfe8f47cb1435615f04af46124debfd5a89fedf9 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Thu, 21 Mar 2024 18:00:09 +0000 Subject: [PATCH v26 2/4] Add test module for verifying read from WAL buffers --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + .../modules/read_wal_from_buffers/.gitignore | 4 + .../modules/read_wal_from_buffers/Makefile | 23 ++++++ .../modules/read_wal_from_buffers/meson.build | 33 +++++++++ .../read_wal_from_buffers--1.0.sql | 14 ++++ .../read_wal_from_buffers.c | 54 ++++++++++++++ .../read_wal_from_buffers.control | 4 + .../read_wal_from_buffers/t/001_basic.pl | 74 +++++++++++++++++++ 9 files changed, 208 insertions(+) create mode 100644 src/test/modules/read_wal_from_buffers/.gitignore create mode 100644 src/test/modules/read_wal_from_buffers/Makefile create mode 100644 src/test/modules/read_wal_from_buffers/meson.build create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control create mode 100644 src/test/modules/read_wal_from_buffers/t/001_basic.pl diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index 1cbd532156..1922b0ed4a 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -12,6 +12,7 @@ SUBDIRS = \ dummy_seclabel \ libpq_pipeline \ plsample \ + read_wal_from_buffers \ spgist_name_ops \ test_bloomfilter \ test_copy_callbacks \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 7c11fb97f2..437fb39ddf 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -10,6 +10,7 @@ subdir('injection_points') subdir('ldap_password_func') subdir('libpq_pipeline') subdir('plsample') +subdir('read_wal_from_buffers') subdir('spgist_name_ops') subdir('ssl_passphrase_callback') subdir('test_bloomfilter') diff --git a/src/test/modules/read_wal_from_buffers/.gitignore b/src/test/modules/read_wal_from_buffers/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/read_wal_from_buffers/Makefile b/src/test/modules/read_wal_from_buffers/Makefile new file mode 100644 index 0000000000..9e57a837f9 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/read_wal_from_buffers/Makefile + +MODULE_big = read_wal_from_buffers +OBJS = \ + $(WIN32RES) \ + read_wal_from_buffers.o +PGFILEDESC = "read_wal_from_buffers - test module to read WAL from WAL buffers" + +EXTENSION = read_wal_from_buffers +DATA = read_wal_from_buffers--1.0.sql + +TAP_TESTS = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/read_wal_from_buffers +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/read_wal_from_buffers/meson.build b/src/test/modules/read_wal_from_buffers/meson.build new file mode 100644 index 0000000000..3fac00d616 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +read_wal_from_buffers_sources = files( + 'read_wal_from_buffers.c', +) + +if host_system == 'windows' + read_wal_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'read_wal_from_buffers', + '--FILEDESC', 'read_wal_from_buffers - test module to read WAL from WAL buffers',]) +endif + +read_wal_from_buffers = shared_module('read_wal_from_buffers', + read_wal_from_buffers_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += read_wal_from_buffers + +test_install_data += files( + 'read_wal_from_buffers.control', + 'read_wal_from_buffers--1.0.sql', +) + +tests += { + 'name': 'read_wal_from_buffers', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_basic.pl', + ], + }, +} diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql new file mode 100644 index 0000000000..82fa097d10 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql @@ -0,0 +1,14 @@ +/* src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION read_wal_from_buffers" to load this file. \quit + +-- +-- read_wal_from_buffers() +-- +-- SQL function to read WAL from WAL buffers. Returns number of bytes read. +-- +CREATE FUNCTION read_wal_from_buffers(IN lsn pg_lsn, IN bytes_to_read int, + bytes_read OUT int) +AS 'MODULE_PATHNAME', 'read_wal_from_buffers' +LANGUAGE C STRICT; diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c new file mode 100644 index 0000000000..9df5c07b4b --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c @@ -0,0 +1,54 @@ +/*-------------------------------------------------------------------------- + * + * read_wal_from_buffers.c + * Test module to read WAL from WAL buffers. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlog.h" +#include "fmgr.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +/* + * SQL function to read WAL from WAL buffers. Returns number of bytes read. + */ +PG_FUNCTION_INFO_V1(read_wal_from_buffers); +Datum +read_wal_from_buffers(PG_FUNCTION_ARGS) +{ + XLogRecPtr startptr = PG_GETARG_LSN(0); + int32 count = PG_GETARG_INT32(1); + Size read; + char *data = palloc0(count); + XLogRecPtr upto = startptr + count; + XLogRecPtr insert_pos = GetXLogInsertRecPtr(); + TimeLineID tli = GetWALInsertionTimeLine(); + + /* + * The requested WAL may be very recent, so wait for any in-progress WAL + * insertions to WAL buffers to finish. + */ + if (upto > insert_pos) + { + XLogRecPtr writtenUpto = WaitXLogInsertionsToFinish(upto); + + upto = Min(upto, writtenUpto); + count = upto - startptr; + } + + read = WALReadFromBuffers(data, startptr, count, tli); + + pfree(data); + + PG_RETURN_INT32(read); +} diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control new file mode 100644 index 0000000000..b14d24751c --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control @@ -0,0 +1,4 @@ +comment = 'Test module to read WAL from WAL buffers' +default_version = '1.0' +module_pathname = '$libdir/read_wal_from_buffers' +relocatable = true diff --git a/src/test/modules/read_wal_from_buffers/t/001_basic.pl b/src/test/modules/read_wal_from_buffers/t/001_basic.pl new file mode 100644 index 0000000000..2360ff1171 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/t/001_basic.pl @@ -0,0 +1,74 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +# Setup a new node. The configuration chosen here minimizes the number +# of arbitrary records that could get generated in a cluster. Enlarging +# checkpoint_timeout avoids noise with checkpoint activity. wal_level +# set to "minimal" avoids random standby snapshot records. Autovacuum +# could also trigger randomly, generating random WAL activity of its own. +# Enlarging wal_writer_delay and wal_writer_flush_after avoid background +# wal flush by walwriter. +my $node = PostgreSQL::Test::Cluster->new("node"); +$node->init; +$node->append_conf( + 'postgresql.conf', + q[wal_level = minimal + autovacuum = off + checkpoint_timeout = '30min' + wal_writer_delay = 10000ms + wal_writer_flush_after = 1GB +]); +$node->start; + +# Setup. +$node->safe_psql('postgres', 'CREATE EXTENSION read_wal_from_buffers;'); + +$node->safe_psql('postgres', 'CREATE TABLE t (c int);'); + +my $result = 0; +my $lsn; +my $to_read; + +# Wait until we read from WAL buffers +for (my $i = 0; $i < 10 * $PostgreSQL::Test::Utils::timeout_default; $i++) +{ + # Get current insert LSN. After this, we generate some WAL which is guranteed + # to be in WAL buffers as there is no other WAL generating activity is + # happening on the server. We then verify if we can read the WAL from WAL + # buffers using this LSN. + $lsn = + $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); + + my $logstart = -s $node->logfile; + + # Generate minimal WAL so that WAL buffers don't get overwritten. + $node->safe_psql('postgres', "INSERT INTO t VALUES ($i);"); + + $to_read = 8192; + + my $res = $node->safe_psql('postgres', + qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) > 0;} + ); + + my $log = $node->log_contains( + "request to flush past end of generated WAL; request .*, current position .*", + $logstart); + + if ($res eq 't' && $log > 0) + { + $result = 1; + last; + } + + usleep(100_000); +} +ok($result, 'waited until WAL is successfully read from WAL buffers'); + +done_testing(); -- 2.34.1