From db027d8f1dcb53ebceef0135287f120acf67cc21 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Thu, 2 Nov 2023 15:36:11 +0000 Subject: [PATCH v14] Allow WAL reading from WAL buffers This commit adds WALRead() the capability to read WAL from WAL buffers when possible. When requested WAL isn't available in WAL buffers, the WAL is read from the WAL file as usual. It relies on WALBufMappingLock so that no one replaces the WAL buffer page that we're reading from. It skips reading from WAL buffers if WALBufMappingLock can't be acquired immediately. In other words, it doesn't wait for WALBufMappingLock to be available. This helps reduce the contention on WALBufMappingLock. This commit benefits the callers of WALRead(), that are walsenders and pg_walinspect. They can now avoid reading WAL from the WAL file (possibly avoiding disk IO). Tests show that the WAL buffers hit ratio stood at 95% for 1 primary, 1 sync standby, 1 async standby, with pgbench --scale=300 --client=32 --time=900. In other words, the walsenders avoided 95% of the time reading from the file/avoided pread system calls: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com This commit also benefits when direct IO is enabled for WAL. Reading WAL from WAL buffers puts back the performance close to that of without direct IO for WAL: https://www.postgresql.org/message-id/CALj2ACV6rS%2B7iZx5%2BoAvyXJaN4AG-djAQeM1mrM%3DYSDkVrUs7g%40mail.gmail.com This commit also paves the way for the following features in future: - Improves synchronous replication performance by replicating directly from WAL buffers. - A opt-in way for the walreceivers to receive unflushed WAL. More details here: https://www.postgresql.org/message-id/20231011224353.cl7c2s222dw3de4j%40awork3.anarazel.de Author: Bharath Rupireddy Reviewed-by: Dilip Kumar, Andres Freund Reviewed-by: Nathan Bossart, Kuntal Ghosh Discussion: https://www.postgresql.org/message-id/CALj2ACXKKK%3DwbiG5_t6dGao5GoecMwRkhr7GjVBM_jg54%2BNa%3DQ%40mail.gmail.com --- src/backend/access/transam/xlog.c | 205 ++++++++++++++++++++++++ src/backend/access/transam/xlogreader.c | 41 ++++- src/include/access/xlog.h | 6 + 3 files changed, 250 insertions(+), 2 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index fdf2ef310b..ff5dccaaa7 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -1753,6 +1753,211 @@ GetXLogBuffer(XLogRecPtr ptr, TimeLineID tli) return cachedPos + ptr % XLOG_BLCKSZ; } +/* + * Read WAL from WAL buffers. + * + * Read 'count' bytes of WAL from WAL buffers into 'buf', starting at location + * 'startptr', on timeline 'tli' and return total read bytes. + * + * This function returns quickly in the following cases: + * - When passed-in timeline is different than server's current insertion + * timeline as WAL is always inserted into WAL buffers on insertion timeline. + * + * - When server is in recovery as WAL buffers aren't currently used in + * recovery. + * + * Note that this function reads as much as it can from WAL buffers, meaning, + * it may not read all the requested 'count' bytes. Caller must be aware of + * this and deal with it. + * + * Note that this function is not available for frontend code as WAL buffers is + * an internal mechanism to the server. + */ +Size +XLogReadFromBuffers(XLogReaderState *state, + XLogRecPtr startptr, + TimeLineID tli, + Size count, + char *buf) +{ + XLogRecPtr ptr; + XLogRecPtr cur_lsn; + Size nbytes; + Size ntotal; + Size nbatch; + char *batchstart; + + if (RecoveryInProgress()) + return 0; + + if (tli != GetWALInsertionTimeLine()) + return 0; + + Assert(!XLogRecPtrIsInvalid(startptr)); + + cur_lsn = GetFlushRecPtr(NULL); + if (unlikely(startptr > cur_lsn)) + elog(ERROR, "WAL start LSN %X/%X specified for reading from WAL buffers must be less than current database system WAL LSN %X/%X", + LSN_FORMAT_ARGS(startptr), LSN_FORMAT_ARGS(cur_lsn)); + + if (!IsWALRecordAvailableInXLogBuffers(startptr)) + return 0; + + /* + * Holding WALBufMappingLock ensures inserters don't overwrite this value + * while we are reading it. We try to acquire it in shared mode so that + * the concurrent WAL readers are also allowed. We try to do as less work + * as possible while holding the lock to not impact concurrent WAL writers + * much. We quickly exit to not cause any contention, if the lock isn't + * immediately available. + */ + if (!LWLockConditionalAcquire(WALBufMappingLock, LW_SHARED)) + return 0; + + ptr = startptr; + nbytes = count; /* Total bytes requested to be read by caller. */ + ntotal = 0; /* Total bytes read. */ + nbatch = 0; /* Bytes to be read in single batch. */ + batchstart = NULL; /* Location to read from for single batch. */ + + while (nbytes > 0) + { + XLogRecPtr expectedEndPtr; + XLogRecPtr endptr; + int idx; + char *page; + char *data; + XLogPageHeader phdr; + + idx = XLogRecPtrToBufIdx(ptr); + expectedEndPtr = ptr; + expectedEndPtr += XLOG_BLCKSZ - ptr % XLOG_BLCKSZ; + endptr = XLogCtl->xlblocks[idx]; + + /* Requested WAL isn't available in WAL buffers. */ + if (expectedEndPtr != endptr) + break; + + /* + * We found WAL buffer page containing given XLogRecPtr. Get starting + * address of the page and a pointer to the right location of given + * XLogRecPtr in that page. + */ + page = XLogCtl->pages + idx * (Size) XLOG_BLCKSZ; + data = page + ptr % XLOG_BLCKSZ; + + /* + * The fact that we acquire WALBufMappingLock while reading the WAL + * buffer page itself guarantees that no one else initializes it or + * makes it ready for next use in AdvanceXLInsertBuffer(). However, we + * need to ensure that we are not reading a page that just got + * initialized. For this, we look at the needed page header. + */ + phdr = (XLogPageHeader) page; + + /* Return, if WAL buffer page doesn't look valid. */ + if (!(phdr->xlp_magic == XLOG_PAGE_MAGIC && + phdr->xlp_pageaddr == (ptr - (ptr % XLOG_BLCKSZ)) && + phdr->xlp_tli == tli)) + break; + + /* + * Note that we don't perform all page header checks here to avoid + * extra work in production builds; callers will anyway do those + * checks extensively. However, in an assert-enabled build, we perform + * all the checks here and raise an error if failed. + */ +#ifdef USE_ASSERT_CHECKING + if (unlikely(state != NULL && + !XLogReaderValidatePageHeader(state, (endptr - XLOG_BLCKSZ), + (char *) phdr))) + { + if (state->errormsg_buf[0]) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg_internal("%s", state->errormsg_buf))); + else + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg_internal("could not read WAL from WAL buffers"))); + } +#endif + + /* Count what is wanted, not the whole page. */ + if ((data + nbytes) <= (page + XLOG_BLCKSZ)) + { + /* All the bytes are in one page. */ + nbatch += nbytes; + ntotal += nbytes; + nbytes = 0; + } + else + { + Size navailable; + + /* + * All the bytes are not in one page. Deduce available bytes on + * the current page, count them and continue to look for remaining + * bytes. + */ + navailable = XLOG_BLCKSZ - (data - page); + Assert(navailable > 0 && navailable <= nbytes); + ptr += navailable; + nbytes -= navailable; + nbatch += navailable; + ntotal += navailable; + } + + /* + * We avoid multiple memcpy calls while reading WAL. Note that we + * memcpy what we have counted so far whenever we are wrapping around + * WAL buffers (because WAL buffers are organized as cirucular array + * of pages) and continue to look for remaining WAL. + */ + if (batchstart == NULL) + { + /* Mark where the data in WAL buffers starts from. */ + batchstart = data; + } + + /* + * We are wrapping around WAL buffers, so read what we have counted so + * far. + */ + if (idx == XLogCtl->XLogCacheBlck) + { + Assert(batchstart != NULL); + Assert(nbatch > 0); + + memcpy(buf, batchstart, nbatch); + buf += nbatch; + + /* Reset for next batch. */ + batchstart = NULL; + nbatch = 0; + } + } + + /* Read what we have counted so far. */ + Assert(nbatch <= ntotal); + if (batchstart != NULL && nbatch > 0) + memcpy(buf, batchstart, nbatch); + + LWLockRelease(WALBufMappingLock); + + /* We never read more than what the caller has asked for. */ + Assert(ntotal <= count); + +#ifdef WAL_DEBUG + if (XLOG_DEBUG) + ereport(DEBUG1, + (errmsg_internal("read %zu bytes out of %zu bytes from WAL buffers for given start LSN %X/%X, timeline ID %u", + ntotal, count, LSN_FORMAT_ARGS(startptr), tli))); +#endif + + return ntotal; +} + /* * Converts a "usable byte position" to XLogRecPtr. A usable byte position * is the position starting from the beginning of WAL, excluding all WAL diff --git a/src/backend/access/transam/xlogreader.c b/src/backend/access/transam/xlogreader.c index e0baa86bd3..5820c5eedc 100644 --- a/src/backend/access/transam/xlogreader.c +++ b/src/backend/access/transam/xlogreader.c @@ -1473,8 +1473,10 @@ err: * Returns true if succeeded, false if an error occurs, in which case * 'errinfo' receives error details. * - * XXX probably this should be improved to suck data directly from the - * WAL buffers when possible. + * When possible, this function reads data directly from WAL buffers. When + * requested WAL isn't available in WAL buffers, the WAL is read from the WAL + * file as usual. The callers may avoid reading WAL from the WAL file thus + * reducing read system calls or even disk IOs. */ bool WALRead(XLogReaderState *state, @@ -1484,6 +1486,41 @@ WALRead(XLogReaderState *state, char *p; XLogRecPtr recptr; Size nbytes; +#ifndef FRONTEND + Size nread; +#endif + +#ifndef FRONTEND + + /* + * Try reading WAL from WAL buffers. Frontend code has no idea of WAL + * buffers. + */ + nread = XLogReadFromBuffers(state, startptr, tli, count, buf); + + Assert(nread >= 0); + + /* + * Check if we have read fully (hit), partially (partial hit) or nothing + * (miss) from WAL buffers. If we have read either partially or nothing, + * then continue to read the remaining bytes the usual way, that is, read + * from WAL file. + * + * XXX: It might be worth to expose WAL buffer read stats. + */ + if (count == nread) + return true; /* Buffer hit, so return. */ + else if (count > nread) + { + /* + * Buffer partial hit, so reset the state to count the read bytes and + * continue. + */ + buf += nread; + startptr += nread; + count -= nread; + } +#endif p = buf; recptr = startptr; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 35235010e6..0e6a3d4264 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -251,6 +251,12 @@ extern XLogRecPtr GetLastImportantRecPtr(void); extern void SetWalWriterSleeping(bool sleeping); +extern Size XLogReadFromBuffers(struct XLogReaderState *state, + XLogRecPtr startptr, + TimeLineID tli, + Size count, + char *buf); + /* * Routines used by xlogrecovery.c to call back into xlog.c during recovery. */ -- 2.34.1