On Wed, Feb 22, 2023 at 6:55 PM John Naylor
<john.nay...@enterprisedb.com> wrote:
>
>
> On Wed, Feb 22, 2023 at 3:29 PM Masahiko Sawada <sawada.m...@gmail.com> wrote:
> >
> > On Wed, Feb 22, 2023 at 4:35 PM John Naylor
> > <john.nay...@enterprisedb.com> wrote:
> > >
> > > I don't think any vacuum calls in regression tests would stress any of
> > > this code very much, so it's not worth carrying the old way forward. I
> > > was thinking of only doing this as a short-time sanity check for testing
> > > a real-world workload.
> >
> > I guess that It would also be helpful at least until the GA release.
> > People will be able to test them easily on their workloads or their
> > custom test scenarios.
>
> That doesn't seem useful to me. If we've done enough testing to reassure us
> the new way always gives the same answer, the old way is not needed at commit
> time. If there is any doubt it will always give the same answer, then the
> whole patchset won't be committed.
True. Even if we're done enough testing we cannot claim there is no
bug. My idea is to make the bug investigation easier but on
reflection, it seems not the best idea given this purpose. Instead, it
seems to be better to add more necessary assertions. What do you think
about the attached patch? Please note that it also includes the
changes for minimum memory requirement.
Regards,
--
Masahiko Sawada
Amazon Web Services: https://aws.amazon.com
diff --git a/src/backend/access/common/tidstore.c
b/src/backend/access/common/tidstore.c
index 9360520482..fc20e58a95 100644
--- a/src/backend/access/common/tidstore.c
+++ b/src/backend/access/common/tidstore.c
@@ -75,6 +75,14 @@ typedef uint64 offsetbm;
#define LOWER_OFFSET_NBITS 6 /* log(sizeof(offsetbm), 2) */
#define LOWER_OFFSET_MASK ((1 << LOWER_OFFSET_NBITS) - 1)
+/*
+ * The minimum amount of memory required by TidStore is 2MB, the current
minimum
+ * valid value for the maintenance_work_mem GUC. This is required to allocate
the
+ * DSA initial segment, 1MB, and some meta data. This number is applied also to
+ * the local TidStore cases for simplicity.
+ */
+#define TIDSTORE_MIN_MEMORY (2 * 1024 * 1024L) /* 2MB */
+
/* A magic value used to identify our TidStore. */
#define TIDSTORE_MAGIC 0x826f6a10
@@ -101,7 +109,7 @@ typedef struct TidStoreControl
/* These values are never changed after creation */
size_t max_bytes; /* the maximum bytes a TidStore can use
*/
- int max_off; /* the maximum offset number */
+ OffsetNumber max_off; /* the maximum offset number */
int max_off_nbits; /* the number of bits required for
offset
* numbers */
int upper_off_nbits; /* the number of bits of offset
numbers
@@ -174,10 +182,17 @@ static inline tidkey encode_tid(TidStore *ts, ItemPointer
tid, offsetbm *off_bit
* The radix tree for storage is allocated in DSA area is 'area' is non-NULL.
*/
TidStore *
-TidStoreCreate(size_t max_bytes, int max_off, dsa_area *area)
+TidStoreCreate(size_t max_bytes, OffsetNumber max_off, dsa_area *area)
{
TidStore *ts;
+ Assert(max_off <= MaxOffsetNumber);
+
+ /* Sanity check for the max_bytes */
+ if (max_bytes < TIDSTORE_MIN_MEMORY)
+ elog(ERROR, "memory for TidStore must be at least %ld, but %zu
provided",
+ TIDSTORE_MIN_MEMORY, max_bytes);
+
ts = palloc0(sizeof(TidStore));
/*
@@ -192,8 +207,8 @@ TidStoreCreate(size_t max_bytes, int max_off, dsa_area
*area)
* In local TidStore cases, the radix tree uses slab allocators for
each kind
* of node class. The most memory consuming case while adding Tids
associated
* with one page (i.e. during TidStoreSetBlockOffsets()) is that we
allocate a new
- * slab block for a new radix tree node, which is approximately 70kB.
Therefore,
- * we deduct 70kB from the max_bytes.
+ * slab block for a new radix tree node, which is approximately 70kB at
most.
+ * Therefore, we deduct 70kB from the max_bytes.
*
* In shared cases, DSA allocates the memory segments big enough to
follow
* a geometric series that approximately doubles the total DSA size (see
@@ -378,6 +393,7 @@ TidStoreSetBlockOffsets(TidStore *ts, BlockNumber blkno,
OffsetNumber *offsets,
const int nkeys = UINT64CONST(1) << ts->control->upper_off_nbits;
Assert(!TidStoreIsShared(ts) || ts->control->magic == TIDSTORE_MAGIC);
+ Assert(BlockNumberIsValid(blkno));
bitmaps = palloc(sizeof(offsetbm) * nkeys);
key = prev_key = key_base;
@@ -386,6 +402,8 @@ TidStoreSetBlockOffsets(TidStore *ts, BlockNumber blkno,
OffsetNumber *offsets,
{
offsetbm off_bit;
+ Assert(offsets[i] <= ts->control->max_off);
+
/* encode the tid to a key and partial offset */
key = encode_blk_off(ts, blkno, offsets[i], &off_bit);
@@ -452,6 +470,8 @@ TidStoreIsMember(TidStore *ts, ItemPointer tid)
offsetbm off_bit;
bool found;
+ Assert(ItemPointerIsValid(tid));
+
key = encode_tid(ts, tid, &off_bit);
if (TidStoreIsShared(ts))
@@ -535,6 +555,7 @@ TidStoreIterateNext(TidStoreIter *iter)
while (tidstore_iter(iter, &key, &off_bitmap))
{
BlockNumber blkno = key_get_blkno(iter->ts, key);
+ Assert(BlockNumberIsValid(blkno));
if (BlockNumberIsValid(output->blkno) && output->blkno != blkno)
{
@@ -586,6 +607,7 @@ TidStoreNumTids(TidStore *ts)
num_tids = ts->control->num_tids;
LWLockRelease(&ts->control->lock);
+ Assert(num_tids >= 0);
return num_tids;
}
diff --git a/src/include/access/tidstore.h b/src/include/access/tidstore.h
index 66f0fdd482..d1cc93cbb6 100644
--- a/src/include/access/tidstore.h
+++ b/src/include/access/tidstore.h
@@ -30,7 +30,7 @@ typedef struct TidStoreIterResult
OffsetNumber offsets[FLEXIBLE_ARRAY_MEMBER];
} TidStoreIterResult;
-extern TidStore *TidStoreCreate(size_t max_bytes, int max_off, dsa_area *dsa);
+extern TidStore *TidStoreCreate(size_t max_bytes, OffsetNumber max_off,
dsa_area *dsa);
extern TidStore *TidStoreAttach(dsa_area *dsa, dsa_pointer handle);
extern void TidStoreDetach(TidStore *ts);
extern void TidStoreDestroy(TidStore *ts);
