Changeset: 216dfc9aa75e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=216dfc9aa75e
Modified Files:
sql/storage/bat/bat_logger.c
sql/storage/store.c
Branch: hot-snapshot
Log Message:
Parse BBP.dir to find the bats that need copying
diffs (265 lines):
diff --git a/sql/storage/bat/bat_logger.c b/sql/storage/bat/bat_logger.c
--- a/sql/storage/bat/bat_logger.c
+++ b/sql/storage/bat/bat_logger.c
@@ -884,7 +884,7 @@ bl_find_table_value(const char *tabnam,
* That part of the file must remain unchanged until the plan is executed.
*/
static void
-snapshot_lazy_copy_file(stream *plan, char *name, long extent)
+snapshot_lazy_copy_file(stream *plan, const char *name, long extent)
{
mnstr_printf(plan, "c %ld %s\n", extent, name);
}
@@ -910,6 +910,8 @@ snapshot_immediate_copy_file(stream *pla
}
size = (long)statbuf.st_size;
+ // TODO we should really do this in pieces,
+ // not allocate memory for the whole file
buf = GDKmalloc(size);
if (!buf) {
GDKerror("malloc failed");
@@ -971,57 +973,166 @@ snapshot_wal(stream *plan, const char *d
snapshot_immediate_copy_file(plan, log_file, log_file + strlen(db_dir)
+ 1);
snprintf(log_file, sizeof(log_file), "%s%s." LLFMT, bat_logger->dir,
LOGFILE, bat_logger->id);
- FILE *f = getFile(log);
- long pos = ftell(f);
- if (pos < 0) {
- GDKerror("ftell failed on %s: %s", log_file, strerror(errno));
+ long extent = getFileSize(log);
+
+ snapshot_lazy_copy_file(plan, log_file, extent);
+
+ return GDK_SUCCEED;
+}
+
+/* If `path` exists, lazy copy it with name `name`.
+ * Otherwise, if `alt_path` exists, lazy copy it with name `alt_name`.
+ *
+ * If `mandatory` is set, it is an error for both files not to exist.
+ *
+ * This interface is slightly messy but of all I tried this gives the most
+ * readable code.
+ */
+static gdk_return
+snapshot_one_heap(stream *plan, bool mandatory, const char *path, const char
*name, const char *alt_path, const char *alt_name)
+{
+ struct stat statbuf;
+ fprintf(stderr, "#cp %s=%s\n", name, path);
+
+ if (stat(path, &statbuf) == 0) {
+ snapshot_lazy_copy_file(plan, name, statbuf.st_size);
+ return GDK_SUCCEED;
+ }
+ if (errno != ENOENT) {
+ GDKerror("Error stat'ing %s: %s", path, strerror(errno));
return GDK_FAIL;
}
-
- // there is some unflushed data in that stream so not all bytes
reported by ftell are
- // actually on disk. Is it safe to flush it? I don't know.
- mnstr_flush(log);
- snapshot_lazy_copy_file(plan, log_file, pos);
+ if (stat(alt_path, &statbuf) == 0) {
+ snapshot_lazy_copy_file(plan, alt_name, statbuf.st_size);
+ return GDK_SUCCEED;
+ }
+ if (errno != ENOENT) {
+ GDKerror("Error stat'ing %s: %s", alt_path, strerror(errno));
+ return GDK_FAIL;
+ }
+
+ if (mandatory) {
+ GDKerror("One of %s and %s must exist", path, alt_path);
+ return GDK_FAIL;
+ }
return GDK_SUCCEED;
}
-/* Add plan entries for all persistent BATs */
+/* Add plan entry for the heaps of one BAT.
+ * path_buffer points at a buffer containing the initial part of
+ * the absolute path of the heap file, for example /tmp/mydatabase/bat/07/726.
+ * This function attempts to add suffixes such as .tail, .theap etc
+ * and copies those files if they exist.
+ * local_part_index is the number of byte to skip to get to the local
+ * part of the filename, in this example skipping the "/tmp/mydatabase/"
+ *
+ * Note: path_buffer must have room for this function to append the suffixes
+ * in-place!
+ */
+static gdk_return
+snapshot_one_bat(stream *plan, char *path_buffer, char *alt_path_buffer,
size_t local_part_index)
+{
+ static const char *suffixes[] = { "M.tail", "O.theap", "O.tvheap",
"O.torderidx", "O.timprint", NULL };
+ //TODO check the above
+ gdk_return ret = GDK_FAIL;
+ char *tail = path_buffer + strlen(path_buffer);
+ char *alt_tail = alt_path_buffer + strlen(alt_path_buffer);
+
+ for (const char **suf = &suffixes[0]; *suf; suf++) {
+ bool mandatory;
+ switch (**suf) {
+ case 'M':
+ mandatory = true;
+ break;
+ case 'O':
+ mandatory = false;
+ break;
+ default:
+ GDKfatal("%s does not start with either M or
O", *suf);
+ }
+ strcpy(tail, *suf + 1);
+ strcpy(alt_tail, *suf + 1);
+ ret = snapshot_one_heap(
+ plan, mandatory,
+ path_buffer, path_buffer + local_part_index,
+ alt_path_buffer, alt_path_buffer + local_part_index);
+ if (ret != GDK_SUCCEED)
+ goto end;
+ }
+
+ ret = GDK_SUCCEED;
+end:
+ *tail = '\0';
+ *alt_tail = '\0';
+ return ret;
+}
+/* Add plan entries for all persistent BATs by looping over the BBP.dir.
+ * Also include the BBP.dir itself.
+ */
static gdk_return
snapshot_bats(stream *plan, const char *db_dir)
{
char bbpdir[FILENAME_MAX];
- bat active_bats;
- gdk_return ret;
+ stream *cat = NULL;
+ char line[1024];
+ int gdk_version;
+ char heapfile[FILENAME_MAX];
+ char alt_heapfile[FILENAME_MAX];
+ size_t heapfile_pos, alt_heapfile_pos;
+ int id;
+ gdk_return ret = GDK_FAIL;
- active_bats = getBBPsize();
-
- snprintf(bbpdir, sizeof(bbpdir), "%s%c%s%c%s", db_dir, DIR_SEP, BAKDIR,
DIR_SEP, "BBP.dir");
+ snprintf(bbpdir, FILENAME_MAX, "%s/%s/%s", db_dir, BAKDIR, "BBP.dir");
ret = snapshot_immediate_copy_file(plan, bbpdir, bbpdir +
strlen(db_dir) + 1);
if (ret == GDK_FAIL)
goto end;
- for (bat id = 1; id < active_bats; id++) {
- if (BBP_status(id) & BBPPERSISTENT) {
- BAT *b = BBP_desc(id);
- snapshot_lazy_copy_heap(plan, &b->theap);
- if (b->tvheap)
- snapshot_lazy_copy_heap(plan, b->tvheap);
- if (b->torderidx)
- snapshot_lazy_copy_heap(plan, b->torderidx);
-
- // Not sure why I commented this out
- // if (b->thash)
- // snapshot_lazy_copy_heap(plan, &b->thash->heap);
-
- // Hmm.. the definition of b->timprints not available
here so
- // b->timprints->heap is unreachable. Hopefully the
system can
- //reconstruct the imprints on demand.
- }
+ // Open the catalog and parse the header
+ cat = open_rastream(bbpdir);
+ if (cat == NULL) {
+ GDKerror("Could not open %s for reading", bbpdir);
+ goto end;
+ }
+ if (mnstr_readline(cat, line, sizeof(line)) < 0) {
+ GDKerror("Could not read first line of %s", bbpdir);
+ goto end;
+ }
+ if (sscanf(line, "BBP.dir, GDKversion %d", &gdk_version) != 1) {
+ GDKerror("Invalid first line of %s", bbpdir);
+ goto end;
+ }
+ if (gdk_version != 25122) { // do not hardcode this
+ GDKerror("Cannot snapshot this gdk version");
+ goto end;
+ }
+ if (mnstr_readline(cat, line, sizeof(line)) < 0) {
+ GDKerror("Couldn't skip the second line of %s", bbpdir);
+ goto end;
+ }
+ if (mnstr_readline(cat, line, sizeof(line)) < 0) {
+ GDKerror("Couldn't skip the third line of %s", bbpdir);
+ goto end;
}
+ heapfile_pos = snprintf(heapfile, FILENAME_MAX, "%s/%s/%s/", db_dir,
BAKDIR, BATDIR);
+ alt_heapfile_pos = snprintf(alt_heapfile, FILENAME_MAX, "%s/%s/",
db_dir, BATDIR);
+ while (mnstr_readline(cat, line, sizeof(line)) > 0) {
+ if (sscanf(line, "%d %*s %*s %s", &id, heapfile + heapfile_pos)
!= 2) {
+ GDKerror("Couldn't parse %s line: %s", bbpdir, line);
+ goto end;
+ }
+ strcpy(alt_heapfile + alt_heapfile_pos, heapfile +
heapfile_pos);
+ snapshot_one_bat(plan, heapfile, alt_heapfile, strlen(db_dir) +
1);
+ }
+
+
end:
+ if (cat) {
+ close_stream(cat);
+ }
+ (void)snapshot_lazy_copy_heap;
return ret;
}
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2466,10 +2466,17 @@ static gdk_return
tar_copy_data(stream *tarfile, const char *path, time_t mtime, stream
*contents, ssize_t size)
{
gdk_return ret = GDK_FAIL;
+ ssize_t file_size = getFileSize(contents);
const ssize_t bufsize = 64 * 1024;
char *buf = malloc(bufsize);
ssize_t nbytes;
+ fprintf(stderr, "#need to copy %ld/%ld bytes of component %s\n", size,
file_size, path);
+ if (file_size < size) {
+ fprintf(stderr, "#adjusting amount to copy to %ld\n",
file_size);
+ size = file_size;
+ }
+
if (!buf) {
GDKerror("could not allocate buffer");
goto end;
@@ -2488,7 +2495,6 @@ tar_copy_data(stream *tarfile, const cha
chunk -= chunk % TAR_BLOCK_SIZE;
assert(chunk > 0);
assert(chunk % TAR_BLOCK_SIZE == 0);
- fprintf(stderr, "#copying %ld/%ld bytes of component %s\n",
chunk, size, path);
nbytes = mnstr_read(contents, buf, 1, chunk);
if (nbytes != chunk) {
GDKerror("Read only %ld/%ld bytes of component %s: %s",
nbytes, chunk, path, mnstr_error(contents));
@@ -2500,7 +2506,6 @@ tar_copy_data(stream *tarfile, const cha
goto end;
}
size -= chunk;
- fprintf(stderr, "# %ld left\n", size);
}
if (size > 0) {
@@ -2515,7 +2520,6 @@ tar_copy_data(stream *tarfile, const cha
GDKerror("Only wrote %ld/%ld bytes of block of
component %s to tar file: %s", nbytes, size, path, mnstr_error(tarfile));
goto end;
}
- fprintf(stderr, "#tail written\n");
}
ret = GDK_SUCCEED;
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
