Changeset: 7170f478bfa1 for MonetDB
URL: http://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=7170f478bfa1
Modified Files:
clients/Tests/exports.stable.out
gdk/gdk_logger.c
geom/monetdb5/geom_upgrade.c
sql/test/testdb-upgrade/Tests/dump.stable.out
sql/test/testdb-upgrade/Tests/dump.stable.out.Windows
Branch: geo
Log Message:
Fixed geom upgrade code.
Upgrading BAT with WKB type must happen before the WAL is processed,
since processing the WAL may append to existing WKB BATs or create new
ones.
Also see the comment.
diffs (truncated from 356 to 300 lines):
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -363,6 +363,8 @@ geomcatalogfix_fptr geomcatalogfix_get(v
void geomcatalogfix_set(geomcatalogfix_fptr);
geomsqlfix_fptr geomsqlfix_get(void);
void geomsqlfix_set(geomsqlfix_fptr);
+int geomversion_get(void);
+void geomversion_set(void);
bat getBBPsize(void);
char *get_bin_path(void);
int gettimeofday(struct timeval *tv, int *ignore_zone);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -1124,6 +1124,8 @@ logger_commit(logger *lg)
return bm_commit(lg);
}
+static void geom_upgrade_wkb(void);
+
static gdk_return
check_version(logger *lg, FILE *fp)
{
@@ -1145,6 +1147,8 @@ check_version(logger *lg, FILE *fp)
return GDK_FAIL;
}
+ if (geomisoldversion)
+ geom_upgrade_wkb();
} else
lg->postfuncp = NULL; /* don't call */
if (fgetc(fp) != '\n' || /* skip \n */
@@ -2848,8 +2852,135 @@ geomversion_set(void)
{
geomisoldversion = 1;
}
-int
-geomversion_get(void)
+int geomversion_get(void)
{
return geomisoldversion;
}
+
+/* "Danger, Will Robinson".
+ *
+ * Upgrade the Well-known Binary (WKB) from older geom versions to the
+ * one in current use. This function should only be called when the
+ * geom module has been loaded, and it should be called before the
+ * Write-ahead Log (WAL) is processed. The WAL may create new BATs
+ * with the WKB type, or append values to an existing BAT. In the
+ * first case it is hard, and in the second impossible, to upgrade the
+ * BAT later.
+ *
+ * This function is located here, since it needs to be called early
+ * (as discussed), and because it calls functions that are GDK only.
+ * There is a little knowledge about the MonetDB WKB type, but nothing
+ * about the internals of the type.
+ *
+ * All errors are fatal.
+ */
+static void
+geom_upgrade_wkb(void)
+{
+ bat bid, bbpsize = getBBPsize();
+ BATstore *bs;
+ BAT *b;
+ int typewkb = ATOMindex("wkb");
+ int utypewkb = ATOMunknown_find("wkb");
+ const char *nme, *bnme;
+ char filename[64];
+ Heap h1, h2;
+ const var_t *restrict old;
+ var_t *restrict new;
+ BUN i;
+ struct old_wkb {
+ int len;
+ char data[FLEXIBLE_ARRAY_MEMBER];
+ } *owkb;
+ struct new_wkb {
+ int len;
+ int srid;
+ char data[FLEXIBLE_ARRAY_MEMBER];
+ } *nwkb;
+ char *oldname, *newname;
+
+ assert(typewkb > 0);
+ for (bid = 1; bid < bbpsize; bid++) {
+ if ((bs = BBP_desc(bid)) == NULL)
+ continue; /* not a valid BAT */
+ b = (BAT *) bs; /* bit of a hack: BATstore contents not known
*/
+ if ((b->T->type != typewkb && b->T->type != utypewkb) ||
+ b->batCount == 0)
+ continue; /* nothing to do for this BAT */
+ assert(b->T->vheap);
+ assert(b->T->width == SIZEOF_VAR_T);
+
+ nme = BBP_physical(bid);
+ if ((bnme = strrchr(nme, DIR_SEP)) == NULL)
+ bnme = nme;
+ else
+ bnme++;
+ snprintf(filename, sizeof(filename), "BACKUP%c%s", DIR_SEP,
bnme);
+ if ((oldname = GDKfilepath(b->T->heap.farmid, BATDIR, nme,
"tail")) == NULL ||
+ (newname = GDKfilepath(b->T->heap.farmid, BAKDIR, bnme,
"tail")) == NULL ||
+ GDKcreatedir(newname) != GDK_SUCCEED ||
+ rename(oldname, newname) < 0)
+ GDKfatal("geom_upgrade_wkb: cannot make backup of
%s.tail\n", nme);
+ GDKfree(oldname);
+ GDKfree(newname);
+ if ((oldname = GDKfilepath(b->T->vheap->farmid, BATDIR, nme,
"theap")) == NULL ||
+ (newname = GDKfilepath(b->T->vheap->farmid, BAKDIR, bnme,
"theap")) == NULL ||
+ rename(oldname, newname) < 0)
+ GDKfatal("geom_upgrade_wkb: cannot make backup of
%s.theap\n", nme);
+ GDKfree(oldname);
+ GDKfree(newname);
+
+ h1 = b->T->heap;
+ h1.filename = NULL;
+ h1.base = NULL;
+ h1.dirty = 0;
+ h2 = *b->T->vheap;
+ h2.filename = NULL;
+ h2.base = NULL;
+ h2.dirty = 0;
+
+ /* load old heaps */
+ if (HEAPload(&h1, filename, "tail", 0) != GDK_SUCCEED ||
+ HEAPload(&h2, filename, "theap", 0) != GDK_SUCCEED)
+ GDKfatal("geom_upgrade_wkb: cannot load old heaps for
BAT %d\n", bid);
+ /* create new heaps */
+ if ((b->T->heap.filename = GDKfilepath(NOFARM, NULL, nme,
"tail")) == NULL ||
+ (b->T->vheap->filename = GDKfilepath(NOFARM, NULL, nme,
"theap")) == NULL)
+ GDKfatal("geom_upgrade_wkb: out of memory\n");
+ if (HEAPalloc(&b->T->heap, b->batCapacity, SIZEOF_VAR_T) !=
GDK_SUCCEED)
+ GDKfatal("geom_upgrade_wkb: cannot allocate heap\n");
+ b->T->heap.dirty = TRUE;
+ b->T->heap.free = h1.free;
+ HEAP_initialize(b->T->vheap, b->batCapacity, 0, (int)
sizeof(var_t));
+ if (b->T->vheap->base == NULL)
+ GDKfatal("geom_upgrade_wkb: cannot allocate heap\n");
+ b->T->vheap->parentid = bid;
+
+ /* do the conversion */
+ b->T->heap.dirty = TRUE;
+ b->T->vheap->dirty = TRUE;
+ old = (const var_t *) h1.base + BUNfirst(b);
+ new = (var_t *) Tloc(b, BUNfirst(b));
+ for (i = 0; i < b->batCount; i++) {
+ int len;
+ owkb = (struct old_wkb *) (h2.base + old[i]);
+ if ((len = owkb->len) == ~0)
+ len = 0;
+ if ((new[i] = HEAP_malloc(b->T->vheap, offsetof(struct
new_wkb, data) + len)) == 0)
+ GDKfatal("geom_upgrade_wkb: cannot allocate
heap space\n");
+ nwkb = (struct new_wkb *) (b->T->vheap->base + (new[i]
<< GDK_VARSHIFT));
+ nwkb->len = owkb->len;
+ nwkb->srid = 0;
+ if (len > 0)
+ memcpy(nwkb->data, owkb->data, len);
+ }
+ HEAPfree(&h1, 0);
+ HEAPfree(&h2, 0);
+ HEAPsave(&b->T->heap, nme, "tail");
+ HEAPfree(&b->T->heap, 0);
+ HEAPsave(b->T->vheap, nme, "theap");
+ HEAPfree(b->T->vheap, 0);
+ }
+ if (TMcommit() != GDK_SUCCEED)
+ GDKfatal("geom_upgrade_wkb: cannot commit changes\n");
+}
diff --git a/geom/monetdb5/geom_upgrade.c b/geom/monetdb5/geom_upgrade.c
--- a/geom/monetdb5/geom_upgrade.c
+++ b/geom/monetdb5/geom_upgrade.c
@@ -116,8 +116,8 @@ int
geom_catalog_upgrade(void *lg, int EC_GEOM, int EC_EXTERNAL, int olddb)
{
/* Do the updates needed for the new geom module */
- BAT *ct, *cnt, *cd, *cnd, *cs, *cns, *cn, *ctid, *ti, *tn, *ts, *si,
*sn, *g;
- BATiter cti, cdi, csi, cni, ctidi, tsi, tni, sni, gi;
+ BAT *ct, *cnt, *cd, *cnd, *cs, *cns, *cn, *ctid, *ti, *tn, *ts, *si,
*sn;
+ BATiter cti, cdi, csi;
char *s = "sys", n[64];
BUN p,q;
char *nt[] = {"types_id", "types_systemname", "types_sqlname",
"types_digits", "types_scale", "types_radix", "types_eclass",
"types_schema_id"};
@@ -142,17 +142,12 @@ geom_catalog_upgrade(void *lg, int EC_GE
cs = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_columns_type_scale")));
csi = bat_iterator(cs);
cn = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_columns_name")));
- cni = bat_iterator(cn);
ctid = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_columns_table_id")));
- ctidi = bat_iterator(ctid);
ti = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_tables_id")));
tn = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_tables_name")));
- tni = bat_iterator(tn);
ts = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"_tables_schema_id")));
- tsi = bat_iterator(ts);
si = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"schemas_id")));
sn = BATdescriptor((bat) logger_find_bat(lg, N(n, NULL, s,
"schemas_name")));
- sni = bat_iterator(sn);
cnt = BATnew(TYPE_void, TYPE_str, BATcount(ct), PERSISTENT);
cnd = BATnew(TYPE_void, TYPE_int, BATcount(cd), PERSISTENT);
@@ -166,80 +161,64 @@ geom_catalog_upgrade(void *lg, int EC_GE
BATseqbase(cns, cs->hseqbase);
for(p=BUNfirst(ct), q=BUNlast(ct); p<q; p++) {
- bte isGeom = 0;
char *type = BUNtail(cti, p);
int digits = *(int*)BUNtail(cdi, p);
int scale = *(int*)BUNtail(csi, p);
- char* colname = BUNtail(cni, p);
if (strcmp(toLower(type), "point") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbPoint << 2;
scale = 0; // in the past we did not save the
srid
} else if (strcmp(toLower(type), "linestring") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbLineString << 2;
scale = 0;
} else if (strcmp(toLower(type), "curve") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbLineString << 2;
scale = 0;
} else if (strcmp(toLower(type), "linearring") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbLinearRing << 2;
scale = 0;
} else if (strcmp(toLower(type), "polygon") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbPolygon << 2;
scale = 0;
} else if (strcmp(toLower(type), "surface") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbPolygon << 2;
scale = 0;
} else if (strcmp(toLower(type), "multipoint") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbMultiPoint << 2;
scale = 0;
} else if (strcmp(toLower(type), "multilinestring") ==
0) {
type = "geometry";
- isGeom = 1;
digits = wkbMultiLineString << 2;
scale = 0;
} else if (strcmp(toLower(type), "multicurve") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbMultiLineString << 2;
scale = 0;
} else if (strcmp(toLower(type), "multipolygon") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbMultiPolygon << 2;
scale = 0;
} else if (strcmp(toLower(type), "multisurface") == 0) {
type = "geometry";
- isGeom = 1;
digits = wkbMultiPolygon << 2;
scale = 0;
} else if (strcmp(toLower(type), "geomcollection") ==
0) {
type = "geometry";
- isGeom = 1;
digits = wkbGeometryCollection << 2;
scale = 0;
} else if (strcmp(toLower(type), "geometrycollection")
== 0) {
type = "geometry";
- isGeom = 1;
digits = wkbGeometryCollection << 2;
scale = 0;
} else if (strcmp(toLower(type), "geometry") == 0) {
type = "geometry";
- isGeom = 1;
digits = 0;
scale = 0;
}
@@ -247,47 +226,6 @@ geom_catalog_upgrade(void *lg, int EC_GE
BUNappend(cnt, type, TRUE);
BUNappend(cnd, &digits, TRUE);
BUNappend(cns, &scale, TRUE);
-
- /* The wkb struct has changed. Update the respective
BATs */
- if (isGeom) {
- typedef struct wkb_old {int len; char
data[FLEXIBLE_ARRAY_MEMBER];} wkb_old;
- BAT *gn;
- BUN k,l;
- int table_id, schema_id;
- char *sn, *tblname;
-
- table_id = *(int*)BUNtail(ctidi, p);
- if ((k = BUNfnd(ti, &table_id)) == BUN_NONE)
- return 0;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
