Changeset: 75feab51bebb for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/75feab51bebb
Modified Files:
gdk/gdk.h
gdk/gdk_bbp.c
gdk/gdk_bbp.h
monetdb5/modules/atoms/json.c
Branch: Dec2023
Log Message:
First pass at json storage upgrade
This is still work in progress. The logic for doing the actual upgrade
is currently disabled.
diffs (227 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -755,7 +755,8 @@ typedef struct {
#define GDKLIBRARY_TAILN 061043U /* first in Jul2021: str offset heaps
names don't take width into account */
#define GDKLIBRARY_HASHASH 061044U /* first in Jul2021: hashash bit in
string heaps */
#define GDKLIBRARY_HSIZE 061045U /* first in Jan2022: heap "size" values
*/
-#define GDKLIBRARY 061046U /* first in Sep2022 */
+#define GDKLIBRARY_JSON 061046U /* first in Dec2023: json storage
changes*/
+#define GDKLIBRARY 061047U /* first in Dec2023 */
/* The batRestricted field indicates whether a BAT is readonly.
* we have modes: BAT_WRITE = all permitted
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -1002,6 +1002,7 @@ BBPheader(FILE *fp, int *lineno, bat *bb
return 0;
}
if (bbpversion != GDKLIBRARY &&
+ bbpversion != GDKLIBRARY_JSON &&
bbpversion != GDKLIBRARY_HSIZE &&
bbpversion != GDKLIBRARY_HASHASH &&
bbpversion != GDKLIBRARY_TAILN &&
@@ -1474,6 +1475,91 @@ movestrbats(void)
}
#endif
+#ifdef GDKLIBRARY_JSON
+gdk_return
+BBPjson_upgrade(json_storage_conversion fixJSONStorage) {
+ (void) fixJSONStorage;
+#if 0
+ bat bid;
+ BAT *b;
+ bat *cmlst;
+ int cnt = 1;
+ int JSON_type = ATOMindex("json");
+
+ if ((cmlst = GDKmalloc(ATOMIC_GET(&BBPsize) * sizeof(bat))) == NULL) {
+ TRC_CRITICAL(GDK, "json storage upgrade: failed to allocate
space");
+ return GDK_FAIL;
+ }
+ cmlst[0] = 0;
+ for (bid = 1; bid < (bat) ATOMIC_GET(&BBPsize); bid++) {
+ if ((b = BBP_desc(bid)) == NULL) {
+ /* not a valid BAT */
+ continue;
+ }
+
+ if (b->ttype < 0) {
+ const char *nme;
+
+ nme = ATOMunknown_name(b->ttype);
+ if (strcmp(nme, "json") != 0)
+ continue;
+ } else if (b->ttype != JSON_type) {
+ continue;
+ }
+ fprintf(stderr, "Upgrading json bat %d\n", bid);
+
+ BAT *b = BATdescriptor(bid);
+ BAT *newb;
+ BATiter bi;
+ struct canditer ci;
+ oid x;
+ str out = NULL;
+
+ newb = COLnew(0, b->ttype, b->batCapacity, PERSISTENT);
+ if (newb == NULL) {
+ TRC_CRITICAL(GDK, "json storage upgrade: new bat
creation failed");
+ return GDK_FAIL;
+ }
+
+ canditer_init(&ci, b, NULL);
+ bi = bat_iterator(b);
+ for (BUN i = 0; i < ci.ncand; i++) {
+ x = canditer_next(&ci);
+ const char *cs = BUNtvar(bi, x);
+ if (!strNil(cs)) {
+ if(fixJSONStorage(&out, &cs) != GDK_SUCCEED) {
+ TRC_CRITICAL(GDK, "could not convert
json string for %s", cs);
+ return GDK_FAIL;
+ }
+ if (BUNappend(newb, out, false) != GDK_SUCCEED)
{
+ TRC_CRITICAL(GDK, "json storage
upgrade: appending value to bat failed");
+ GDKfree(out);
+ return GDK_FAIL;
+ }
+ GDKfree(out);
+ out = NULL;
+ }
+ }
+ bat_iterator_end(&bi);
+ if (BBPsave(newb) != GDK_SUCCEED) {
+ return GDK_FAIL;
+ }
+ cmlst[cnt++] = newb->batCacheid;
+ BBPunfix(newb->batCacheid);
+ BBPunfix(bid);
+ }
+ if (TMsubcommit_list(cmlst, NULL, cnt, -1, -1) != GDK_SUCCEED) {
+ GDKfree(cmlst);
+ return GDK_FAIL;
+ }
+ GDKfree(cmlst);
+#endif // 0
+ /* We did the upgrade, remove the signal file */
+ GDKunlink(0, BATDIR, "jsonupgradeneeded", NULL);
+ return GDK_SUCCEED;
+}
+#endif
+
static bool
BBPtrim(bool aggressive)
{
@@ -1815,6 +1901,38 @@ BBPinit(bool allow_hge_upgrade)
return res;
#endif
+#ifdef GDKLIBRARY_JSON
+ if (bbpversion < GDKLIBRARY) {
+ char *jsonupgradestr;
+ if (GDKinmemory(0)) {
+ jsonupgradestr = NULL;
+ } else {
+ if ((jsonupgradestr = GDKfilepath(0, BATDIR,
"jsonupgradeneeded", NULL)) == NULL) {
+ TRC_CRITICAL(GDK, "GDKfilepath failed\n");
+ ATOMIC_SET(&GDKdebug, dbg);
+ return GDK_FAIL;
+ }
+
+ /* create signal file that we need to upgrade
+ * stored json strings. This will be performed
+ * by an upgrade function in the GDK that will
+ * be called at the end of the json module
+ * initialzation with a callback that actually
+ * knows how to perform the upgrade. */
+ int fd = MT_open(jsonupgradestr, O_WRONLY | O_CREAT);
+ if (fd < 0) {
+ TRC_CRITICAL(GDK, "cannot create signal file
jsonupgradeneeded");
+ GDKfree(jsonupgradestr);
+ ATOMIC_SET(&GDKdebug, dbg);
+ return GDK_FAIL;
+ }
+
+ close(fd);
+ GDKfree(jsonupgradestr);
+ }
+ }
+#endif
+
if (bbpversion < GDKLIBRARY && TMcommit() != GDK_SUCCEED) {
TRC_CRITICAL(GDK, "TMcommit failed\n");
ATOMIC_SET(&GDKdebug, dbg);
@@ -4298,6 +4416,12 @@ BBPdiskscan(const char *parent, size_t b
if (dent->d_name[0] == '.')
continue; /* ignore .dot files and directories (.
..) */
+#ifdef GDKLIBRARY_JSON
+ if (strncmp(dent->d_name, "jsonupgradeneed", 15) == 0) {
+ continue; /* ignore json upgrade signal file */
+ }
+#endif
+
if (strncmp(dent->d_name, "BBP.", 4) == 0 &&
(strcmp(parent + baseoff, BATDIR) == 0 ||
strncmp(parent + baseoff, BAKDIR, strlen(BAKDIR)) == 0 ||
diff --git a/gdk/gdk_bbp.h b/gdk/gdk_bbp.h
--- a/gdk/gdk_bbp.h
+++ b/gdk/gdk_bbp.h
@@ -91,7 +91,10 @@ gdk_export int BBPrelease(bat b);
gdk_export void BBPkeepref(BAT *b)
__attribute__((__nonnull__(1)));
gdk_export void BBPcold(bat i);
-
+#ifdef GDKLIBRARY_JSON
+typedef gdk_return ((*json_storage_conversion)(char **, const char **));
+gdk_export gdk_return BBPjson_upgrade(json_storage_conversion);
+#endif
#define BBP_status_set(bid, mode) \
ATOMIC_SET(&BBP_record(bid).status, mode)
diff --git a/monetdb5/modules/atoms/json.c b/monetdb5/modules/atoms/json.c
--- a/monetdb5/modules/atoms/json.c
+++ b/monetdb5/modules/atoms/json.c
@@ -573,10 +573,40 @@ JSONisarray(bit *ret, json *js)
return MAL_SUCCEED;
}
+#ifdef GDKLIBRARY_JSON
+static
+gdk_return upgradeJSONStorage(char **out, const char **in) {
+ if (JSONstr2json(out, in) != MAL_SUCCEED) {
+ return GDK_FAIL;
+ }
+ return GDK_SUCCEED;
+}
+
+#endif
+
static str
JSONprelude(void)
{
TYPE_json = ATOMindex("json");
+#ifdef GDKLIBRARY_JSON
+/* Run the gdk upgrade libary function with a callback that
+ * performs the actual upgrade.
+ */
+ char *jsonupgrade;
+ struct stat st;
+ if ((jsonupgrade = GDKfilepath(0, BATDIR, "jsonupgradeneeded", NULL))
== NULL) {
+ throw(MAL, "json.prelude", SQLSTATE(HY013) MAL_MALLOC_FAIL); //
Fix exception reason
+ }
+ int r = stat(jsonupgrade, &st);
+ if (r == 0) {
+ /* The file exists so we need to run the upgrade code */
+ if (BBPjson_upgrade(upgradeJSONStorage) != GDK_SUCCEED) {
+ GDKfree(jsonupgrade);
+ throw(MAL, "json.prelude", SQLSTATE(HY013)
MAL_MALLOC_FAIL); // Fix exception reason
+ }
+ }
+ GDKfree(jsonupgrade);
+#endif
return MAL_SUCCEED;
}
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
