[PATCH] btree_gist: fix union implementation for variable length columns

[Pavel Raiskup]

Hi hackers,

while I tried to debug 'gcc -fstack-protector -O3' problems in [1], I noticed
that gbt_var_union() mistreats the first vector element.  Patch is attached.

[1] https://bugzilla.redhat.com/show_bug.cgi?id=1544349
Pavel

>From 4e4f0fe8d2f74a85f37abdf095ab8aecf9329596 Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <prais...@redhat.com>
Date: Wed, 4 Jul 2018 11:39:38 +0200
Subject: [PATCH] Fix btree_gist "union" for variable length types

The gbt_var_union() did not expect that first key in the entryvec
could represent leaf node, while it many times is.  So try call
f_l2n() even for the first element, same as we do for all the
other elements in gbt_var_bin_union().

This bug could create wrong indexes, so we it's suggested to
REINDEX.

diff --git a/contrib/btree_gist/btree_bit.c b/contrib/btree_gist/btree_bit.c
index 2225244ded..432b788204 100644
*** a/contrib/btree_gist/btree_bit.c
--- b/contrib/btree_gist/btree_bit.c
***************
*** 75,81 **** gbt_bitcmp(const void *a, const void *b, Oid collation, FmgrInfo *flinfo)
  static bytea *
  gbt_bit_xfrm(bytea *leaf)
  {
! 	bytea	   *out = leaf;
  	int			sz = VARBITBYTES(leaf) + VARHDRSZ;
  	int			padded_sz = INTALIGN(sz);
  
--- 75,81 ----
  static bytea *
  gbt_bit_xfrm(bytea *leaf)
  {
! 	bytea	   *out;
  	int			sz = VARBITBYTES(leaf) + VARHDRSZ;
  	int			padded_sz = INTALIGN(sz);
  
diff --git a/contrib/btree_gist/btreeindex 670c879e77..6d8299d9b5 100644
*** a/contrib/btree_gist/btree_utils_var.c
--- b/contrib/btree_gist/btree_utils_var.c
***************
*** 230,252 **** gbt_var_node_truncate(const GBT_VARKEY *node, int32 cpf_length, const gbtree_vin
  }
  
  
  
  void
  gbt_var_bin_union(Datum *u, GBT_VARKEY *e, Oid collation,
  				  const gbtree_vinfo *tinfo, FmgrInfo *flinfo)
  {
! 	GBT_VARKEY_R eo = gbt_var_key_readable(e);
  	GBT_VARKEY_R nr;
  
- 	if (eo.lower == eo.upper)	/* leaf */
- 	{
- 		GBT_VARKEY *tmp;
- 
- 		tmp = gbt_var_leaf2node(e, tinfo, flinfo);
- 		if (tmp != e)
- 			eo = gbt_var_key_readable(tmp);
- 	}
- 
  	if (DatumGetPointer(*u))
  	{
  		GBT_VARKEY_R ro = gbt_var_key_readable((GBT_VARKEY *) DatumGetPointer(*u));
--- 230,258 ----
  }
  
  
+ static GBT_VARKEY_R
+ gbt_var_key_readable_resolved(GBT_VARKEY *key, const gbtree_vinfo *tinfo,
+ 							  FmgrInfo *flinfo)
+ {
+ 	GBT_VARKEY_R out = gbt_var_key_readable(key);
+ 	if (out.lower == out.upper) /* leaf */
+ 	{
+ 		GBT_VARKEY *tmp;
+ 		tmp = gbt_var_leaf2node(key, tinfo, flinfo);
+ 		if (tmp != key)
+ 			out = gbt_var_key_readable(tmp);
+ 	}
+ 	return out;
+ }
+ 
  
  void
  gbt_var_bin_union(Datum *u, GBT_VARKEY *e, Oid collation,
  				  const gbtree_vinfo *tinfo, FmgrInfo *flinfo)
  {
! 	GBT_VARKEY_R eo = gbt_var_key_readable_resolved(e, tinfo, flinfo);
  	GBT_VARKEY_R nr;
  
  	if (DatumGetPointer(*u))
  	{
  		GBT_VARKEY_R ro = gbt_var_key_readable((GBT_VARKEY *) DatumGetPointer(*u));
***************
*** 333,339 **** gbt_var_union(const GistEntryVector *entryvec, int32 *size, Oid collation,
  	*size = sizeof(GBT_VARKEY);
  
  	cur = (GBT_VARKEY *) DatumGetPointer(entryvec->vector[0].key);
! 	rk = gbt_var_key_readable(cur);
  	out = PointerGetDatum(gbt_var_key_copy(&rk));
  
  	for (i = 1; i < numranges; i++)
--- 339,345 ----
  	*size = sizeof(GBT_VARKEY);
  
  	cur = (GBT_VARKEY *) DatumGetPointer(entryvec->vector[0].key);
! 	rk = gbt_var_key_readable_resolved(cur, tinfo, flinfo);
  	out = PointerGetDatum(gbt_var_key_copy(&rk));
  
  	for (i = 1; i < numranges; i++)