From d348c029ca3db228753b8db9cea214951eced4ee Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Thu, 25 Jan 2024 06:54:33 +0000 Subject: [PATCH v20] Add test module for verifying WAL read from WAL buffers This commit adds a test module to verify WAL read from WAL buffers. Author: Bharath Rupireddy Reviewed-by: Dilip Kumar Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com --- src/test/modules/Makefile | 1 + src/test/modules/meson.build | 1 + .../modules/read_wal_from_buffers/.gitignore | 4 ++ .../modules/read_wal_from_buffers/Makefile | 23 ++++++++ .../modules/read_wal_from_buffers/meson.build | 33 ++++++++++++ .../read_wal_from_buffers--1.0.sql | 14 +++++ .../read_wal_from_buffers.c | 41 +++++++++++++++ .../read_wal_from_buffers.control | 4 ++ .../read_wal_from_buffers/t/001_basic.pl | 52 +++++++++++++++++++ 9 files changed, 173 insertions(+) create mode 100644 src/test/modules/read_wal_from_buffers/.gitignore create mode 100644 src/test/modules/read_wal_from_buffers/Makefile create mode 100644 src/test/modules/read_wal_from_buffers/meson.build create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c create mode 100644 src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control create mode 100644 src/test/modules/read_wal_from_buffers/t/001_basic.pl diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile index e32c8925f6..4eba0fa2e2 100644 --- a/src/test/modules/Makefile +++ b/src/test/modules/Makefile @@ -12,6 +12,7 @@ SUBDIRS = \ dummy_seclabel \ libpq_pipeline \ plsample \ + read_wal_from_buffers \ spgist_name_ops \ test_bloomfilter \ test_copy_callbacks \ diff --git a/src/test/modules/meson.build b/src/test/modules/meson.build index 397e0906e6..f0b53eced7 100644 --- a/src/test/modules/meson.build +++ b/src/test/modules/meson.build @@ -32,6 +32,7 @@ subdir('test_resowner') subdir('test_rls_hooks') subdir('test_shm_mq') subdir('test_slru') +subdir('read_wal_from_buffers') subdir('unsafe_tests') subdir('worker_spi') subdir('xid_wraparound') diff --git a/src/test/modules/read_wal_from_buffers/.gitignore b/src/test/modules/read_wal_from_buffers/.gitignore new file mode 100644 index 0000000000..5dcb3ff972 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/read_wal_from_buffers/Makefile b/src/test/modules/read_wal_from_buffers/Makefile new file mode 100644 index 0000000000..9e57a837f9 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/Makefile @@ -0,0 +1,23 @@ +# src/test/modules/read_wal_from_buffers/Makefile + +MODULE_big = read_wal_from_buffers +OBJS = \ + $(WIN32RES) \ + read_wal_from_buffers.o +PGFILEDESC = "read_wal_from_buffers - test module to read WAL from WAL buffers" + +EXTENSION = read_wal_from_buffers +DATA = read_wal_from_buffers--1.0.sql + +TAP_TESTS = 1 + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/read_wal_from_buffers +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/read_wal_from_buffers/meson.build b/src/test/modules/read_wal_from_buffers/meson.build new file mode 100644 index 0000000000..3fac00d616 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/meson.build @@ -0,0 +1,33 @@ +# Copyright (c) 2024, PostgreSQL Global Development Group + +read_wal_from_buffers_sources = files( + 'read_wal_from_buffers.c', +) + +if host_system == 'windows' + read_wal_from_buffers_sources += rc_lib_gen.process(win32ver_rc, extra_args: [ + '--NAME', 'read_wal_from_buffers', + '--FILEDESC', 'read_wal_from_buffers - test module to read WAL from WAL buffers',]) +endif + +read_wal_from_buffers = shared_module('read_wal_from_buffers', + read_wal_from_buffers_sources, + kwargs: pg_test_mod_args, +) +test_install_libs += read_wal_from_buffers + +test_install_data += files( + 'read_wal_from_buffers.control', + 'read_wal_from_buffers--1.0.sql', +) + +tests += { + 'name': 'read_wal_from_buffers', + 'sd': meson.current_source_dir(), + 'bd': meson.current_build_dir(), + 'tap': { + 'tests': [ + 't/001_basic.pl', + ], + }, +} diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql new file mode 100644 index 0000000000..82fa097d10 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql @@ -0,0 +1,14 @@ +/* src/test/modules/read_wal_from_buffers/read_wal_from_buffers--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION read_wal_from_buffers" to load this file. \quit + +-- +-- read_wal_from_buffers() +-- +-- SQL function to read WAL from WAL buffers. Returns number of bytes read. +-- +CREATE FUNCTION read_wal_from_buffers(IN lsn pg_lsn, IN bytes_to_read int, + bytes_read OUT int) +AS 'MODULE_PATHNAME', 'read_wal_from_buffers' +LANGUAGE C STRICT; diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c new file mode 100644 index 0000000000..da841da3f0 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c @@ -0,0 +1,41 @@ +/*-------------------------------------------------------------------------- + * + * read_wal_from_buffers.c + * Test module to read WAL from WAL buffers. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/test/modules/read_wal_from_buffers/read_wal_from_buffers.c + * ------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/xlog.h" +#include "fmgr.h" +#include "utils/pg_lsn.h" + +PG_MODULE_MAGIC; + +/* + * SQL function to read WAL from WAL buffers. Returns number of bytes read. + */ +PG_FUNCTION_INFO_V1(read_wal_from_buffers); +Datum +read_wal_from_buffers(PG_FUNCTION_ARGS) +{ + XLogRecPtr startptr = PG_GETARG_LSN(0); + int32 bytes_to_read = PG_GETARG_INT32(1); + Size bytes_read = 0; + char *data = palloc0(bytes_to_read); + + bytes_read = XLogReadFromBuffers(data, startptr, + (Size) bytes_to_read, + GetWALInsertionTimeLine()); + + pfree(data); + + PG_RETURN_INT32(bytes_read); +} diff --git a/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control new file mode 100644 index 0000000000..b14d24751c --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/read_wal_from_buffers.control @@ -0,0 +1,4 @@ +comment = 'Test module to read WAL from WAL buffers' +default_version = '1.0' +module_pathname = '$libdir/read_wal_from_buffers' +relocatable = true diff --git a/src/test/modules/read_wal_from_buffers/t/001_basic.pl b/src/test/modules/read_wal_from_buffers/t/001_basic.pl new file mode 100644 index 0000000000..e1773da2c8 --- /dev/null +++ b/src/test/modules/read_wal_from_buffers/t/001_basic.pl @@ -0,0 +1,52 @@ +# Copyright (c) 2021-2023, PostgreSQL Global Development Group + +use strict; +use warnings; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +my $node = PostgreSQL::Test::Cluster->new('test'); + +$node->init; + +# Ensure nobody interferes with us so that the WAL in WAL buffers don't get +# overwritten while running tests. +$node->append_conf( + 'postgresql.conf', qq( +autovacuum = off +checkpoint_timeout = 1h +wal_writer_delay = 10000ms +wal_writer_flush_after = 1GB +)); +$node->start; + +# Setup. +$node->safe_psql('postgres', 'CREATE EXTENSION read_wal_from_buffers;'); + +# Get current insert LSN. After this, we generate some WAL which is guranteed +# to be in WAL buffers as there is no other WAL generating activity is +# happening on the server. We then verify if we can read the WAL from WAL +# buffers using this LSN. +my $lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); + +# Generate minimal WAL so that WAL buffers don't get overwritten. +$node->safe_psql('postgres', + "CREATE TABLE t (c int); INSERT INTO t VALUES (1);"); + +# Check if WAL is successfully read from WAL buffers. +my $to_read = 8192; +my $result = $node->safe_psql('postgres', + qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) > 0;}); +is($result, 't', "WAL is successfully read from WAL buffers"); + +# Check with a WAL that doesn't yet exist i.e., 16MB starting from current +# flush LSN. +$lsn = $node->safe_psql('postgres', 'SELECT pg_current_wal_flush_lsn()+16777216;'); +$to_read = 8192; +$result = $node->safe_psql('postgres', + qq{SELECT read_wal_from_buffers(lsn := '$lsn', bytes_to_read := $to_read) = 0;}); +is($result, 't', "WAL that doesn't yet exist is not read from WAL buffers"); + +done_testing(); -- 2.34.1