Hi Thomas,
I've fixed the warnings.
Thomas Munro <thomas.mu...@gmail.com> 于2019年7月5日周五 下午12:21写道：
> On Thu, Jun 20, 2019 at 1:51 AM Binguo Bao <djydew...@gmail.com> wrote:
> > Hi hackers!
> > This proposal aims to provide the ability to de-TOAST a fully TOAST'd
> and compressed field using an iterator and then update the appropriate
> parts of the code to use the iterator where possible instead of
> de-TOAST'ing and de-compressing the entire value. Examples where this can
> be helpful include using position() from the beginning of the value, or
> doing a pattern or substring match.
> >
> > de-TOAST iterator overview:
> > 1. The caller requests the slice of the attribute value from the
> de-TOAST iterator.
> > 2. The de-TOAST iterator checks if there is a slice available in the
> output buffer, if there is, return the result directly,
> > otherwise goto the step3.
> > 3. The de-TOAST iterator checks if there is the slice available in the
> input buffer, if there is, goto step44. Otherwise,
> > call fetch_datum_iterator to fetch datums from disk to input buffer.
> > 4. If the data in the input buffer is compressed, extract some data from
> the input buffer to the output buffer until the caller's
> > needs are met.
> >
> > I've implemented the prototype and apply it to the position() function
> to test performance.
>
> Hi Binguo,
>
> Interesting work, and nice performance improvements so far. Just by
> the way, the patch currently generates warnings:
>
> https://travis-ci.org/postgresql-cfbot/postgresql/builds/554345719
>
> --
> Thomas Munro
> https://enterprisedb.com
>
From 0ac012867be04b409bfc4e67306fd9f3e5b4785a Mon Sep 17 00:00:00 2001
From: BBG <djydew...@gmail.com>
Date: Tue, 4 Jun 2019 22:56:42 +0800
Subject: [PATCH] de-TOASTing using a iterator
---
src/backend/access/heap/tuptoaster.c | 489 +++++++++++++++++++++++++++++++++++
src/backend/utils/adt/varlena.c | 48 ++--
src/include/access/tuptoaster.h | 92 +++++++
3 files changed, 613 insertions(+), 16 deletions(-)
diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c
index 55d6e91..07bb69e 100644
--- a/src/backend/access/heap/tuptoaster.c
+++ b/src/backend/access/heap/tuptoaster.c
@@ -83,6 +83,13 @@ static int toast_open_indexes(Relation toastrel,
static void toast_close_indexes(Relation *toastidxs, int num_indexes,
LOCKMODE lock);
static void init_toast_snapshot(Snapshot toast_snapshot);
+static FetchDatumIterator create_fetch_datum_iterator(struct varlena *attr);
+static bool free_fetch_datum_iterator(FetchDatumIterator iter);
+static int32 fetch_datum_iterate(FetchDatumIterator iter);
+static void init_toast_buffer(ToastBuffer *buf, int size, bool compressed);
+static bool free_toast_buffer(ToastBuffer *buf);
+static int32 pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest,
+ DetoastIterator iter, int32 length);
/* ----------
@@ -347,6 +354,145 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
/* ----------
+ * create_detoast_iterator -
+ *
+ * Initialize detoast iterator.
+ * ----------
+ */
+DetoastIterator create_detoast_iterator(struct varlena *attr) {
+ struct varatt_external toast_pointer;
+ DetoastIterator iterator = NULL;
+ if (VARATT_IS_EXTERNAL_ONDISK(attr))
+ {
+ /*
+ * This is an externally stored datum --- create fetch datum iterator
+ */
+ iterator = (DetoastIterator) palloc0(sizeof(DetoastIteratorData));
+ iterator->fetch_datum_iterator = create_fetch_datum_iterator(attr);
+ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
+ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
+ {
+ /* If it's compressed, prepare buffer for raw data */
+ iterator->buf = (ToastBuffer *) palloc0(sizeof(ToastBuffer));
+ init_toast_buffer(iterator->buf, toast_pointer.va_rawsize, false);
+ iterator->source = NULL;
+ iterator->ctrlc = 0;
+ iterator->compressed = true;
+ iterator->done = false;
+ }
+ else
+ {
+ iterator->buf = iterator->fetch_datum_iterator->buf;
+ iterator->source = NULL;
+ iterator->ctrlc = 0;
+ iterator->compressed = false;
+ iterator->done = false;
+ }
+ }
+ else if (VARATT_IS_EXTERNAL_INDIRECT(attr))
+ {
+ /*
+ * This is an indirect pointer --- dereference it
+ */
+ struct varatt_indirect redirect;
+
+ VARATT_EXTERNAL_GET_POINTER(redirect, attr);
+ attr = (struct varlena *) redirect.pointer;
+
+ /* nested indirect Datums aren't allowed */
+ Assert(!VARATT_IS_EXTERNAL_INDIRECT(attr));
+
+ /* recurse in case value is still extended in some other way */
+ iterator = create_detoast_iterator(attr);
+
+ }
+ else if (VARATT_IS_COMPRESSED(attr))
+ {
+ /*
+ * This is a compressed value inside of the main tuple
+ */
+ iterator = (DetoastIterator) palloc0(sizeof(DetoastIteratorData));
+ iterator->fetch_datum_iterator = NULL;
+ iterator->source = palloc0(sizeof(ToastBuffer));
+ iterator->source->buf = (const char*) attr;
+ iterator->source->position = TOAST_COMPRESS_RAWDATA(attr);
+ iterator->source->limit = (char *)attr + VARSIZE(attr);
+ iterator->source->capacity = iterator->source->limit;
+
+ iterator->buf = palloc0(sizeof(ToastBuffer));
+ init_toast_buffer(iterator->buf, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ, false);
+
+ iterator->ctrlc = 0;
+ iterator->compressed = true;
+ iterator->done = false;
+ }
+
+ return iterator;
+}
+
+
+/* ----------
+ * free_detoast_iterator -
+ *
+ * Free the memory space occupied by the de-Toast iterator.
+ * ----------
+ */
+bool free_detoast_iterator(DetoastIterator iter) {
+ if (iter == NULL)
+ {
+ return false;
+ }
+ if (iter->buf != iter->fetch_datum_iterator->buf)
+ {
+ free_toast_buffer(iter->buf);
+ }
+ free_fetch_datum_iterator(iter->fetch_datum_iterator);
+ free_toast_buffer(iter->source);
+ pfree(iter);
+ return true;
+}
+
+
+/* ----------
+ * detoast_iterate -
+ *
+ * Iterate through the toasted value referenced by iterator.
+ *
+ * As long as there is another slice in compression or external storage,
+ * detoast it into toast buffer in iterator, and return available slice length.
+ * Return -1 when no more data.
+ * ----------
+ */
+extern int32 detoast_iterate(int32 length, DetoastIterator iter)
+{
+ if (iter == NULL)
+ {
+ elog(ERROR, "detoast_iterate shouln't be called for NULL iterator");
+ }
+
+ if (iter->buf->limit - iter->buf->position >= length || iter->done)
+ {
+ return iter->buf->limit - iter->buf->position;
+ }
+
+ if (iter->fetch_datum_iterator != NULL)
+ {
+ ToastBuffer *buf = iter->fetch_datum_iterator->buf;
+ FetchDatumIterator fetch_iter = iter->fetch_datum_iterator;
+ while(buf->limit - buf->position < length && !fetch_iter->done) {
+ fetch_datum_iterate(fetch_iter);
+ }
+ if (iter->compressed) {
+ return pglz_decompress_iterate(buf, iter->buf, iter, length);
+ }
+ return iter->buf->limit - iter->buf->position;
+ }
+
+ return pglz_decompress_iterate(iter->source, iter->buf, iter, length);
+}
+
+
+/* ----------
* toast_raw_datum_size -
*
* Return the raw (detoasted) size of a varlena datum
@@ -2409,3 +2555,346 @@ init_toast_snapshot(Snapshot toast_snapshot)
InitToastSnapshot(*toast_snapshot, snapshot->lsn, snapshot->whenTaken);
}
+
+
+/* ----------
+ * create_fetch_datum_iterator -
+ *
+ * Initialize fetch datum iterator.
+ * ----------
+ */
+static FetchDatumIterator
+create_fetch_datum_iterator(struct varlena *attr) {
+ int validIndex;
+ FetchDatumIterator iterator;
+
+ if (!VARATT_IS_EXTERNAL_ONDISK(attr))
+ elog(ERROR, "create_fetch_datum_itearator shouldn't be called for non-ondisk datums");
+
+ iterator = (FetchDatumIterator) palloc0(sizeof(FetchDatumIteratorData));
+
+ /* Must copy to access aligned fields */
+ VARATT_EXTERNAL_GET_POINTER(iterator->toast_pointer, attr);
+
+ iterator->ressize = iterator->toast_pointer.va_extsize;
+ iterator->numchunks = ((iterator->ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
+
+ /*
+ * Open the toast relation and its indexes
+ */
+ iterator->toastrel = table_open(iterator->toast_pointer.va_toastrelid, AccessShareLock);
+
+ /* Look for the valid index of the toast relation */
+ validIndex = toast_open_indexes(iterator->toastrel,
+ AccessShareLock,
+ &iterator->toastidxs,
+ &iterator->num_indexes);
+
+ /*
+ * Setup a scan key to fetch from the index by va_valueid
+ */
+ ScanKeyInit(&iterator->toastkey,
+ (AttrNumber) 1,
+ BTEqualStrategyNumber, F_OIDEQ,
+ ObjectIdGetDatum(iterator->toast_pointer.va_valueid));
+
+ /*
+ * Read the chunks by index
+ *
+ * Note that because the index is actually on (valueid, chunkidx) we will
+ * see the chunks in chunkidx order, even though we didn't explicitly ask
+ * for it.
+ */
+
+ init_toast_snapshot(&iterator->SnapshotToast);
+ iterator->toastscan = systable_beginscan_ordered(iterator->toastrel, iterator->toastidxs[validIndex],
+ &iterator->SnapshotToast, 1, &iterator->toastkey);
+
+ iterator->buf = (ToastBuffer *) palloc0(sizeof(ToastBuffer));
+ init_toast_buffer(iterator->buf, iterator->ressize + VARHDRSZ, VARATT_EXTERNAL_IS_COMPRESSED(iterator->toast_pointer));
+
+ iterator->nextidx = 0;
+ iterator->done = false;
+
+ return iterator;
+}
+
+static bool
+free_fetch_datum_iterator(FetchDatumIterator iter)
+{
+ if (iter == NULL)
+ {
+ return false;
+ }
+
+ if (!iter->done)
+ {
+ systable_endscan_ordered(iter->toastscan);
+ toast_close_indexes(iter->toastidxs, iter->num_indexes, AccessShareLock);
+ table_close(iter->toastrel, AccessShareLock);
+ }
+ free_toast_buffer(iter->buf);
+ pfree(iter);
+ return true;
+}
+
+/* ----------
+ * fetch_datum_iterate -
+ *
+ * Iterate through the toasted value referenced by iterator.
+ *
+ * As long as there is another chunk data in compression or external storage,
+ * fetch it into buffer in iterator, and return slice length.
+ * Return -1 when no more data.
+ * ----------
+ */
+static int32
+fetch_datum_iterate(FetchDatumIterator iter) {
+ HeapTuple ttup;
+ TupleDesc toasttupDesc;
+ int32 residx;
+ Pointer chunk;
+ bool isnull;
+ char *chunkdata;
+ int32 chunksize;
+
+ if (iter == NULL)
+ {
+ elog(ERROR, "fetch_datum_iterate shouln't be called for NULL iterator");
+ }
+
+ if (iter->done)
+ {
+ return -1;
+ }
+
+ ttup = systable_getnext_ordered(iter->toastscan, ForwardScanDirection);
+ if (ttup == NULL)
+ {
+ /*
+ * Final checks that we successfully fetched the datum
+ */
+ if (iter->nextidx != iter->numchunks)
+ elog(ERROR, "missing chunk number %d for toast value %u in %s",
+ iter->nextidx,
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+
+ /*
+ * End scan and close relations
+ */
+ systable_endscan_ordered(iter->toastscan);
+ toast_close_indexes(iter->toastidxs, iter->num_indexes, AccessShareLock);
+ table_close(iter->toastrel, AccessShareLock);
+
+ iter->done = true;
+ return -1;
+ }
+
+ /*
+ * Have a chunk, extract the sequence number and the data
+ */
+ toasttupDesc = iter->toastrel->rd_att;
+ residx = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull));
+ Assert(!isnull);
+ chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull));
+ Assert(!isnull);
+ if (!VARATT_IS_EXTENDED(chunk))
+ {
+ chunksize = VARSIZE(chunk) - VARHDRSZ;
+ chunkdata = VARDATA(chunk);
+ }
+ else if (VARATT_IS_SHORT(chunk))
+ {
+ /* could happen due to heap_form_tuple doing its thing */
+ chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
+ chunkdata = VARDATA_SHORT(chunk);
+ }
+ else
+ {
+ /* should never happen */
+ elog(ERROR, "found toasted toast chunk for toast value %u in %s",
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+ chunksize = 0; /* keep compiler quiet */
+ chunkdata = NULL;
+ }
+
+ /*
+ * Some checks on the data we've found
+ */
+ if (residx != iter->nextidx)
+ elog(ERROR, "unexpected chunk number %d (expected %d) for toast value %u in %s",
+ residx, iter->nextidx,
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+ if (residx < iter->numchunks - 1)
+ {
+ if (chunksize != TOAST_MAX_CHUNK_SIZE)
+ elog(ERROR, "unexpected chunk size %d (expected %d) in chunk %d of %d for toast value %u in %s",
+ chunksize, (int) TOAST_MAX_CHUNK_SIZE,
+ residx, iter->numchunks,
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+ }
+ else if (residx == iter->numchunks - 1)
+ {
+ if ((residx * TOAST_MAX_CHUNK_SIZE + chunksize) != iter->ressize)
+ elog(ERROR, "unexpected chunk size %d (expected %d) in final chunk %d for toast value %u in %s",
+ chunksize,
+ (int) (iter->ressize - residx * TOAST_MAX_CHUNK_SIZE),
+ residx,
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+ }
+ else
+ elog(ERROR, "unexpected chunk number %d (out of range %d..%d) for toast value %u in %s",
+ residx,
+ 0, iter->numchunks - 1,
+ iter->toast_pointer.va_valueid,
+ RelationGetRelationName(iter->toastrel));
+
+ /*
+ * Copy the data into proper place in our iterator buffer
+ */
+ memcpy(iter->buf->limit, chunkdata, chunksize);
+ iter->buf->limit += chunksize;
+
+ iter->nextidx++;
+ return chunksize;
+}
+
+
+static void
+init_toast_buffer(ToastBuffer *buf, int32 size, bool compressed) {
+ buf->buf = (const char *) palloc0(size);
+ if (compressed) {
+ SET_VARSIZE_COMPRESSED(buf->buf, size);
+ buf->position = VARDATA_4B_C(buf->buf);
+ }
+ else
+ {
+ SET_VARSIZE(buf->buf, size);
+ buf->position = VARDATA_4B(buf->buf);
+ }
+ buf->limit = VARDATA(buf->buf);
+ buf->capacity = buf->buf + size;
+ buf->buf_size = size;
+}
+
+
+static bool
+free_toast_buffer(ToastBuffer *buf)
+{
+ if (buf == NULL)
+ {
+ return false;
+ }
+
+ pfree((void *)buf->buf);
+ pfree(buf);
+}
+
+
+/* ----------
+ * pglz_decompress_iterate -
+ *
+ * Decompresses source into dest. Returns the number of available bytes
+ * decompressed in the destination buffer.
+ * ----------
+ */
+static int32
+pglz_decompress_iterate(ToastBuffer *source, ToastBuffer *dest, DetoastIterator iter, int32 need_len)
+{
+ const unsigned char *sp;
+ const unsigned char *srcend;
+ unsigned char *dp;
+ unsigned char *destend;
+
+ sp = (const unsigned char *) source->position;
+ srcend = ((const unsigned char *) source->limit);
+ dp = (unsigned char *) dest->limit;
+ destend = (unsigned char *)dest->capacity;
+
+ while (sp < srcend && dp < destend && ((char *)dp - dest->position) < need_len)
+ {
+ /*
+ * Read one control byte and process the next 8 items (or as many as
+ * remain in the compressed input).
+ */
+ unsigned char ctrl;
+ int ctrlc;
+ if (iter->ctrlc != 0) {
+ ctrl = iter->ctrl;
+ ctrlc = iter->ctrlc;
+ }
+ else
+ {
+ ctrl = *sp++;
+ ctrlc = 0;
+ }
+
+
+ for (; ctrlc < 8 && sp < srcend && dp < destend; ctrlc++)
+ {
+
+ if (ctrl & 1)
+ {
+ /*
+ * Otherwise it contains the match length minus 3 and the
+ * upper 4 bits of the offset. The next following byte
+ * contains the lower 8 bits of the offset. If the length is
+ * coded as 18, another extension tag byte tells how much
+ * longer the match really was (0-255).
+ */
+ int32 len;
+ int32 off;
+
+ len = (sp[0] & 0x0f) + 3;
+ off = ((sp[0] & 0xf0) << 4) | sp[1];
+ sp += 2;
+ if (len == 18)
+ len += *sp++;
+
+ /*
+ * Now we copy the bytes specified by the tag from OUTPUT to
+ * OUTPUT. It is dangerous and platform dependent to use
+ * memcpy() here, because the copied areas could overlap
+ * extremely!
+ */
+ len = Min(len, destend - dp);
+ while (len--)
+ {
+ *dp = dp[-off];
+ dp++;
+ }
+ }
+ else
+ {
+ /*
+ * An unset control bit means LITERAL BYTE. So we just copy
+ * one from INPUT to OUTPUT.
+ */
+ *dp++ = *sp++;
+ }
+
+ /*
+ * Advance the control bit
+ */
+ ctrl >>= 1;
+ }
+
+ if (ctrlc < 8) {
+ iter->ctrlc = ctrlc;
+ iter->ctrl = ctrl;
+ }
+ else
+ {
+ iter->ctrlc = 0;
+ }
+ }
+
+ source->position = (char *) sp;
+ dest->limit = (char *) dp;
+ return dest->limit - dest->position;
+}
diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c
index 0864838..e7fab58 100644
--- a/src/backend/utils/adt/varlena.c
+++ b/src/backend/utils/adt/varlena.c
@@ -122,10 +122,10 @@ static text *text_substring(Datum str,
int32 length,
bool length_not_specified);
static text *text_overlay(text *t1, text *t2, int sp, int sl);
-static int text_position(text *t1, text *t2, Oid collid);
+static int text_position(text *t1, text *t2, Oid collid, DetoastIterator iter);
static void text_position_setup(text *t1, text *t2, Oid collid, TextPositionState *state);
-static bool text_position_next(TextPositionState *state);
-static char *text_position_next_internal(char *start_ptr, TextPositionState *state);
+static bool text_position_next(TextPositionState *state, DetoastIterator iter);
+static char *text_position_next_internal(char *start_ptr, TextPositionState *state, DetoastIterator iter);
static char *text_position_get_match_ptr(TextPositionState *state);
static int text_position_get_match_pos(TextPositionState *state);
static void text_position_cleanup(TextPositionState *state);
@@ -1092,10 +1092,19 @@ text_overlay(text *t1, text *t2, int sp, int sl)
Datum
textpos(PG_FUNCTION_ARGS)
{
- text *str = PG_GETARG_TEXT_PP(0);
+ text *str;
+ DetoastIterator iter = create_detoast_iterator((struct varlena *)(DatumGetPointer(PG_GETARG_DATUM(0))));
+ if (iter != NULL)
+ {
+ str = (text *) iter->buf->buf;
+ }
+ else
+ {
+ str = PG_GETARG_TEXT_PP(0);
+ }
text *search_str = PG_GETARG_TEXT_PP(1);
- PG_RETURN_INT32((int32) text_position(str, search_str, PG_GET_COLLATION()));
+ PG_RETURN_INT32((int32) text_position(str, search_str, PG_GET_COLLATION(), iter));
}
/*
@@ -1113,7 +1122,7 @@ textpos(PG_FUNCTION_ARGS)
* functions.
*/
static int
-text_position(text *t1, text *t2, Oid collid)
+text_position(text *t1, text *t2, Oid collid, DetoastIterator iter)
{
TextPositionState state;
int result;
@@ -1122,15 +1131,15 @@ text_position(text *t1, text *t2, Oid collid)
return 0;
text_position_setup(t1, t2, collid, &state);
- if (!text_position_next(&state))
+ if (!text_position_next(&state, iter))
result = 0;
else
result = text_position_get_match_pos(&state);
text_position_cleanup(&state);
+ free_detoast_iterator(iter);
return result;
}
-
/*
* text_position_setup, text_position_next, text_position_cleanup -
* Component steps of text_position()
@@ -1274,7 +1283,7 @@ text_position_setup(text *t1, text *t2, Oid collid, TextPositionState *state)
* is found.
*/
static bool
-text_position_next(TextPositionState *state)
+text_position_next(TextPositionState *state, DetoastIterator iter)
{
int needle_len = state->len2;
char *start_ptr;
@@ -1290,7 +1299,7 @@ text_position_next(TextPositionState *state)
start_ptr = state->str1;
retry:
- matchptr = text_position_next_internal(start_ptr, state);
+ matchptr = text_position_next_internal(start_ptr, state, iter);
if (!matchptr)
return false;
@@ -1338,7 +1347,7 @@ retry:
* match starting at 'start_ptr', or NULL if no match is found.
*/
static char *
-text_position_next_internal(char *start_ptr, TextPositionState *state)
+text_position_next_internal(char *start_ptr, TextPositionState *state, DetoastIterator iter)
{
int haystack_len = state->len1;
int needle_len = state->len2;
@@ -1358,6 +1367,9 @@ text_position_next_internal(char *start_ptr, TextPositionState *state)
hptr = start_ptr;
while (hptr < haystack_end)
{
+ if (iter != NULL) {
+ detoast_iterate(hptr - iter->buf->position + 1, iter);
+ }
if (*hptr == nchar)
return (char *) hptr;
hptr++;
@@ -1371,6 +1383,10 @@ text_position_next_internal(char *start_ptr, TextPositionState *state)
hptr = start_ptr + needle_len - 1;
while (hptr < haystack_end)
{
+ if (iter != NULL) {
+ detoast_iterate(hptr - iter->buf->position + 1, iter);
+ }
+
/* Match the needle scanning *backward* */
const char *nptr;
const char *p;
@@ -4231,7 +4247,7 @@ replace_text(PG_FUNCTION_ARGS)
text_position_setup(src_text, from_sub_text, PG_GET_COLLATION(), &state);
- found = text_position_next(&state);
+ found = text_position_next(&state, NULL);
/* When the from_sub_text is not found, there is nothing to do. */
if (!found)
@@ -4256,7 +4272,7 @@ replace_text(PG_FUNCTION_ARGS)
start_ptr = curr_ptr + from_sub_text_len;
- found = text_position_next(&state);
+ found = text_position_next(&state, NULL);
if (found)
curr_ptr = text_position_get_match_ptr(&state);
}
@@ -4594,7 +4610,7 @@ split_text(PG_FUNCTION_ARGS)
/* identify bounds of first field */
start_ptr = VARDATA_ANY(inputstring);
- found = text_position_next(&state);
+ found = text_position_next(&state, NULL);
/* special case if fldsep not found at all */
if (!found)
@@ -4612,7 +4628,7 @@ split_text(PG_FUNCTION_ARGS)
{
/* identify bounds of next field */
start_ptr = end_ptr + fldsep_len;
- found = text_position_next(&state);
+ found = text_position_next(&state, NULL);
if (found)
end_ptr = text_position_get_match_ptr(&state);
}
@@ -4766,7 +4782,7 @@ text_to_array_internal(PG_FUNCTION_ARGS)
CHECK_FOR_INTERRUPTS();
- found = text_position_next(&state);
+ found = text_position_next(&state, NULL);
if (!found)
{
/* fetch last field */
diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h
index f0aea24..e60815d 100644
--- a/src/include/access/tuptoaster.h
+++ b/src/include/access/tuptoaster.h
@@ -17,6 +17,98 @@
#include "storage/lockdefs.h"
#include "utils/relcache.h"
+#ifndef FRONTEND
+#include "access/genam.h"
+
+/*
+ * TOAST buffer is a producer consumer buffer.
+ *
+ * +--+--+--+--+--+--+--+--+--+--+--+--+--+
+ * | | | | | | | | | | | | | |
+ * +--+--+--+--+--+--+--+--+--+--+--+--+--+
+ * ^ ^ ^ ^
+ * buf position limit capacity
+ *
+ * buf: point to the start of buffer.
+ * position: point to the next char to be consume.
+ * limit: point to the next char to be produce.
+ * capacity: point to the end of buffer.
+ *
+ * Constrains that need to be satisfied:
+ * buf <= position <= limit <= capacity
+ */
+typedef struct ToastBuffer
+{
+ const char *buf;
+ const char *position;
+ char *limit;
+ const char *capacity;
+ int32 buf_size;
+} ToastBuffer;
+
+
+typedef struct FetchDatumIteratorData
+{
+ ToastBuffer *buf;
+ Relation toastrel;
+ Relation *toastidxs;
+ SysScanDesc toastscan;
+ ScanKeyData toastkey;
+ SnapshotData SnapshotToast;
+ struct varatt_external toast_pointer;
+ int32 ressize;
+ int32 nextidx;
+ int32 numchunks;
+ int num_indexes;
+ bool done;
+} FetchDatumIteratorData;
+
+typedef struct FetchDatumIteratorData *FetchDatumIterator;
+
+typedef struct DetoastIteratorData
+{
+ ToastBuffer *buf;
+ FetchDatumIterator fetch_datum_iterator;
+ ToastBuffer *source;
+ unsigned char ctrl;
+ int ctrlc;
+ bool compressed; /* toast value is compressed? */
+ bool done; /* iterator exhausted? */
+} DetoastIteratorData;
+
+typedef struct DetoastIteratorData *DetoastIterator;
+
+/* ----------
+ * create_detoast_iterator -
+ *
+ * Initialize detoast iterator.
+ * ----------
+ */
+extern DetoastIterator create_detoast_iterator(struct varlena *attr);
+
+/* ----------
+ * free_detoast_iterator -
+ *
+ * Free the memory space occupied by the de-Toast iterator.
+ * ----------
+ */
+extern bool free_detoast_iterator(DetoastIterator iter);
+
+/* ----------
+ * detoast_iterate -
+ *
+ * Iterate through the toasted value referenced by iterator.
+ *
+ * As long as there is another slice in compression or external storage,
+ * detoast it into toast buffer in iterator, and return available slice length.
+ * Return -1 when no more data.
+ * ----------
+ */
+extern int32 detoast_iterate(int32 length, DetoastIterator iter);
+
+#endif
+
+
/*
* This enables de-toasting of index entries. Needed until VACUUM is
* smart enough to rebuild indexes from scratch.
--
2.7.4
