Changeset: 625c5d10d35a for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=625c5d10d35a
Added Files:
tools/merovingian/daemon/snapshot.c
tools/merovingian/daemon/snapshot.h
Modified Files:
sql/jdbc/tests/Tests/JdbcClient_create_tables.sql
sql/jdbc/tests/Tests/JdbcClient_inserts_selects.sql
sql/storage/store.c
sql/test/BugTracker-2019/Tests/next-get-value-bulk.Bug-6766.stable.out
sql/test/Tests/50ways.sql
sql/test/Tests/add_column_with_seq.sql
sql/test/Tests/alter-sequence.sql
sql/test/Tests/antiselect.sql
sql/test/Tests/cast_str2num.sql
sql/test/Tests/coalesce.sql
sql/test/Tests/count.sql
sql/test/Tests/crashme.sql
sql/test/Tests/create_in_schema.sql
sql/test/Tests/decimal.sql
sql/test/Tests/decimal2.sql
sql/test/Tests/default.sql
sql/test/Tests/example-delete1.sql
sql/test/Tests/example-delete2.sql
sql/test/Tests/example-delete3.sql
sql/test/Tests/example-drop.sql
sql/test/Tests/example-queries.sql
sql/test/Tests/example-rangeselect.sql
sql/test/Tests/example-transaction.sql
sql/test/Tests/example-update-view.sql
sql/test/Tests/example-update1.sql
sql/test/Tests/example-update2.sql
sql/test/Tests/example-update3.sql
sql/test/Tests/example.sql
sql/test/Tests/function_syntax.sql
sql/test/Tests/identifiers.sql
sql/test/Tests/joins.sql
sql/test/Tests/keys.sql
sql/test/Tests/marcin1.sql
sql/test/Tests/marcin2.sql
sql/test/Tests/null.sql
sql/test/Tests/order_by_simple_exp.sql
sql/test/Tests/orderby_limit.sql
sql/test/Tests/orderby_non_selection_column.sql
sql/test/Tests/procedure_syntax.sql
sql/test/Tests/psm.sql
sql/test/Tests/restart-sequence.sql
sql/test/Tests/round.sql
sql/test/Tests/session_vars.sql
sql/test/Tests/setoptimizer.sql
sql/test/Tests/string.sql
sql/test/Tests/table_function.sql
sql/test/Tests/timezone.sql
sql/test/Tests/zones2.sql
sql/test/VOC/VOCcreate_user.sql
sql/test/VOC/VOCdrop.sql
sql/test/VOC/VOCdrop_user.sql
sql/test/VOC/VOCinsert.sql
sql/test/VOC/VOCmanual_examples.sql
sql/test/VOC/VOCquery.sql
sql/test/bugs/Tests/arith-bug-sf-949228.sql
sql/test/bugs/Tests/autocommit_illegal_statements-bug-sf-953545.sql
sql/test/bugs/Tests/case_cast-bug-sf-1019506.sql
sql/test/bugs/Tests/case_parenthesis-bug-sf-1023056.sql
sql/test/bugs/Tests/cast_bool2int-bug-sf-1000576.sql
sql/test/bugs/Tests/cast_bool2int-bug-sf-951800.sql
sql/test/bugs/Tests/cast_varchar2int-bug-sf-964165.sql
sql/test/bugs/Tests/create_insert_select-bug-sf-900198.sql
sql/test/bugs/Tests/create_insert_select_aggr-bug-00001.sql
sql/test/bugs/Tests/date_parsing-bug-sf-960166.sql
sql/test/bugs/Tests/drop_foreignkey_selfreferencing.sql
sql/test/bugs/Tests/except-union-intersect-bug-sf-1146079.sql
sql/test/bugs/Tests/extract_interval_bug-sf-1274088.sql
sql/test/bugs/Tests/float_arith_cast2int-bug-sf-949255.sql
sql/test/bugs/Tests/floor-bug-sf-967066.sql
sql/test/bugs/Tests/foreignkey_leftjoin-bug-sf-854312.sql
sql/test/bugs/Tests/groupby_having-bug-sf-947600.sql
sql/test/bugs/Tests/groupby_having_charlength-bug-sf-943566.sql
sql/test/bugs/Tests/innerjoin_between-bug-sf-960672.sql
sql/test/bugs/Tests/innerjoin_multiple-bug-sf-943661.sql
sql/test/bugs/Tests/insert_into_groupby-bug-sf-949135.sql
sql/test/bugs/Tests/insert_into_select-bug-sf-978043.sql
sql/test/bugs/Tests/insert_update_arith-bug-sf-900206.sql
sql/test/bugs/Tests/interval_convert_bugs-sf-1274077-1274085.sql
sql/test/bugs/Tests/interval_data_commutative_add_bug-sf-1263928.sql
sql/test/bugs/Tests/interval_parse_bug-sf-1263967.sql
sql/test/bugs/Tests/like-bug-sf-1031435.sql
sql/test/bugs/Tests/like-bug-sf-1043521.sql
sql/test/bugs/Tests/numeric-bug-sf-952758.sql
sql/test/bugs/Tests/object_name-bug-sf-985242.sql
sql/test/bugs/Tests/primary_key-bug-sf-975308.sql
sql/test/bugs/Tests/savepoint_release-bug-sf-953560.sql
sql/test/bugs/Tests/savepoint_rollback-bug-sf-1023652.sql
sql/test/bugs/Tests/savepoint_rollback-bug-sf-953553.sql
sql/test/bugs/Tests/select_constant_from_where_1=1-bug-sf-1019529.sql
sql/test/bugs/Tests/select_orderby_alias-bug-sf-1024615.sql
sql/test/bugs/Tests/selectlist_subquery-sf-1208599.sql
sql/test/bugs/Tests/start_transaction-bug-sf-952755.sql
sql/test/bugs/Tests/subselect_ambigious_columns-bug-sf-949071.sql
sql/test/bugs/Tests/subselect_column-bug-sf-951206.sql
sql/test/bugs/Tests/subselect_multiple_unionall_where_1=1-bug-sf-1005596.sql
sql/test/bugs/Tests/subselect_single_value-bug-sf-941791.sql
sql/test/bugs/Tests/subselect_single_value-bug-sf-962099.sql
sql/test/bugs/Tests/subselect_union-bug-sf-941826.sql
sql/test/bugs/Tests/timestamp_comparison-bug-sf-1158581.sql
sql/test/bugs/Tests/trimtest-bug-sf-1023588.sql
sql/test/bugs/Tests/unicode_varchar-bug-sf-1041324.sql
sql/test/bugs/Tests/union-bug-sf-978045.sql
sql/test/bugs/Tests/union_all-bug-sf-941788.sql
sql/test/bugs/Tests/union_all_where_1=1-bug-sf-936386.sql
sql/test/bugs/Tests/unique-bug-sf-1043512.sql
sql/test/bugs/Tests/update_where-bug-sf-976963.sql
sql/test/bugs/Tests/view_join-bug-sf-990096.sql
sql/test/bugs/Tests/view_selfjoin-bug-sf-937835.sql
sql/test/miscellaneous/Tests/groupby_error.stable.out
sql/test/testdb/Tests/dump-nogeom.sql
sql/test/testdb/Tests/dump.sql
sql/test/testdb/Tests/dump.timeout
sql/test/testdb/Tests/load.timeout
sql/test/testdb/Tests/package-hge.py
sql/test/testdb/Tests/package.py
sql/test/wlcr/Tests/wlr10.py
tools/merovingian/client/monetdb.c
tools/merovingian/daemon/Makefile.ag
tools/merovingian/daemon/controlrunner.c
tools/merovingian/daemon/merovingian.c
tools/merovingian/utils/database.c
tools/merovingian/utils/utils.c
tools/merovingian/utils/utils.h
Branch: default
Log Message:
Merged hot-snapshot branch into default.
diffs (truncated from 1833 to 300 lines):
diff --git a/sql/storage/store.c b/sql/storage/store.c
--- a/sql/storage/store.c
+++ b/sql/storage/store.c
@@ -2554,7 +2554,11 @@ hot_snapshot_write_tar(stream *out, cons
goto end;
}
}
- ret = GDK_SUCCEED;
+
+ // write a trailing block of zeros. If it succeeds, this function
succeeds.
+ char a;
+ a = '\0';
+ ret = tar_write(out, &a, 1);
end:
free(plan);
@@ -2563,13 +2567,50 @@ end:
return ret;
}
+/* Pick a name for the temporary tar file. Make sure it has the same extension
+ * so as not to confuse the streams library.
+ *
+ * This function is not entirely safe as compare to for example mkstemp.
+ */
+static str pick_tmp_name(str filename)
+{
+ str name = GDKmalloc(strlen(filename) + 10);
+ if (name == NULL) {
+ GDKerror("malloc failed");
+ return NULL;
+ }
+ strcpy(name, filename);
+
+ // Look for an extension.
+ // Make sure it's part of the basename
+
+ char *ext = strrchr(name, '.');
+ char *sep = strrchr(name, DIR_SEP);
+ char *slash = strrchr(name, '/'); // on Windows, / and \ both work
+ if (ext != NULL && sep != NULL && sep > ext)
+ ext = NULL;
+ else if (ext != NULL && slash != NULL && slash > ext)
+ ext = NULL;
+
+ if (ext == NULL) {
+ return strcat(name, "..tmp");
+ } else {
+ char *tmp = "..tmp.";
+ size_t tmplen = strlen(tmp);
+ memmove(ext + tmplen, ext, strlen(ext) + 1);
+ memmove(ext, tmp, tmplen);
+ }
+
+ return name;
+}
+
extern lng
store_hot_snapshot(str tarfile)
{
int locked = 0;
lng result = 0;
- char tmppath[FILENAME_MAX];
- char dirpath[FILENAME_MAX];
+ char *tmppath = NULL;
+ char *dirpath = NULL;
int do_remove = 0;
int dir_fd = -1;
stream *tar_stream = NULL;
@@ -2582,7 +2623,10 @@ store_hot_snapshot(str tarfile)
goto end;
}
- snprintf(tmppath, sizeof(tmppath), "%s.tmp", tarfile);
+ tmppath = pick_tmp_name(tarfile);
+ if (tmppath == NULL) {
+ goto end;
+ }
tar_stream = open_wstream(tmppath);
if (!tar_stream) {
GDKerror("Failed to open %s for writing", tmppath);
@@ -2600,8 +2644,13 @@ store_hot_snapshot(str tarfile)
// Call realpath(2) to make the path absolute so it has at least
// one DIR_SEP in it. Realpath requires the file to exist so
// we feed it tmppath rather than tarfile.
- if (realpath(tmppath, dirpath) == NULL) { // ERROR no realpath
- GDKsyserror("couldn't resolve path %s", tarfile);
+ dirpath = GDKmalloc(PATH_MAX);
+ if (dirpath == NULL) {
+ GDKsyserror("malloc failed");
+ goto end;
+ }
+ if (realpath(tmppath, dirpath) == NULL) {
+ GDKsyserror("couldn't resolve path %s: %s", tarfile,
strerror(errno));
goto end;
}
*strrchr(dirpath, DIR_SEP) = '\0';
@@ -2616,7 +2665,7 @@ store_hot_snapshot(str tarfile)
}
// Fsync the directory. Postgres believes this is necessary for
durability.
- if (fsync(dir_fd) < 0) { // ERROR no fsync
+ if (fsync(dir_fd) < 0) {
GDKsyserror("First fsync on %s failed", dirpath);
goto end;
}
@@ -2678,6 +2727,8 @@ store_hot_snapshot(str tarfile)
result = 42;
end:
+ GDKfree(tmppath);
+ GDKfree(dirpath);
if (dir_fd >= 0)
close(dir_fd);
if (locked)
diff --git a/tools/merovingian/client/monetdb.c
b/tools/merovingian/client/monetdb.c
--- a/tools/merovingian/client/monetdb.c
+++ b/tools/merovingian/client/monetdb.c
@@ -58,10 +58,11 @@ command_help(int argc, char *argv[])
if (argc < 2) {
printf("Usage: monetdb [options] command
[command-options-and-arguments]\n");
printf(" where command is one of:\n");
- printf(" create, destroy, lock, release\n");
- printf(" status, start, stop, kill\n");
- printf(" profilerstart, profilerstop\n");
- printf(" set, get, inherit\n");
+ printf(" create, destroy, lock, release,\n");
+ printf(" status, start, stop, kill,\n");
+ printf(" profilerstart, profilerstop,\n");
+ printf(" snapshot,\n");
+ printf(" set, get, inherit,\n");
printf(" discover, help, version\n");
printf(" options can be:\n");
printf(" -q suppress status output\n");
@@ -169,6 +170,39 @@ command_help(int argc, char *argv[])
} else if (strcmp(argv[1], "version") == 0) {
printf("Usage: monetdb version\n");
printf(" prints the version of this monetdb utility\n");
+ } else if (strcmp(argv[1], "snapshot") == 0) {
+ if (argc > 2 && strcmp(argv[2], "list") == 0) {
+ printf("Usage: monetdb snapshot list [<dbname>...]\n");
+ printf(" List snapshots for the given database, or all
databases\n");
+ printf(" if none given.\n");
+ } else if (argc > 2 && strcmp(argv[2], "create") == 0) {
+ printf("Usage: monetdb snapshot create [-t
<targetfile>] <dbname> [<dbname>..]\n");
+ printf(" Take a snapshot of the listed databases.
Unless -t is given, the snapshots\n");
+ printf(" are written to files named\n");
+ printf("
<snapshotdir>/<dbname>_<YYYY><MM><DD>T<HH><MM>UTC.tar.gz.\n");
+ printf("Options:\n");
+ printf(" -t <targetfile> File on the server to write
the snapshot to.\n");
+ } else if (argc > 2 && strcmp(argv[2], "restore") == 0) {
+ printf("Usage: monetdb snapshot restore [-f] <snapid>
[dbname]\n");
+ printf(" Create a database from the given snapshot,
where <snapid> is either\n");
+ printf(" a path on the server or <dbname>@<num> as
produced\n");
+ printf("Options:\n");
+ printf(" -f do not ask for confirmation\n");
+ } else if (argc > 2 && strcmp(argv[2], "destroy") == 0) {
+ printf("Usage: monetdb snapshot destroy [-f]
<snapid>...\n");
+ printf(" monetdb snapshot destroy [-f] -r <N>
<dbname>...\n");
+ printf(" Destroy one or more database snapshots,
identified by a database name\n");
+ printf(" and a sequence number as given by 'monetdb
snapshot list'.\n");
+ printf(" In the first form, the sequence numbers are
part of the <snapid>.\n");
+ printf(" In the second form. <dbname> is a database
name or pattern such as 'staging*'\n");
+ printf(" and N is the number of snapshots to
retain.\n");
+ printf("Options:\n");
+ printf(" -f Do not ask for confirmation\n");
+ printf(" -r Number of snapshots to retain.\n");
+ } else {
+ printf("Usage: monetdb <create|list|restore|destroy>
[arguments]\n");
+ printf(" Manage database snapshots\n");
+ }
} else {
printf("help: unknown command: %s\n", argv[1]);
}
@@ -1658,6 +1692,599 @@ command_profilerstop(int argc, char *arg
simple_command(argc, argv, "profilerstop", "stopped profiler", 1);
}
+/* Snapshot this single database to the given file */
+static void
+snapshot_create_adhoc(sabdb *databases, char *filename) {
+ /* databases is supposed to only hold a single database */
+ assert(databases != NULL);
+ assert(databases->next == NULL);
+
+ char *merocmd = malloc(100 + strlen(filename));
+ sprintf(merocmd, "snapshot create adhoc %s", filename);
+
+ simple_argv_cmd("snapshot", databases, merocmd, NULL, "snapshotting
database");
+
+ free(merocmd);
+}
+
+/* Create automatic snapshots of the given databases */
+static void
+snapshot_create_automatic(sabdb *databases) {
+ simple_argv_cmd("snapshot", databases, "snapshot create automatic",
NULL, "snapshotting database");
+}
+
+/* Comparison function used for qsort */
+static int
+snapshot_enumerate_helper(const void *left, const void *right)
+{
+ const struct snapshot *left_snap = left;
+ const struct snapshot *right_snap = right;
+ int cmp;
+
+ cmp = strcmp(left_snap->dbname, right_snap->dbname);
+ if (cmp != 0)
+ return cmp;
+
+ // Careful! Sort newest to oldest
+ if (left_snap->time < right_snap->time)
+ return +1; // !!
+ if (left_snap->time > right_snap->time)
+ return -1; // !!
+
+ // No preference
+ return 0;
+}
+
+/* Retrieve a list of all snapshots and Store it in the array. */
+static char*
+snapshot_enumerate(struct snapshot **snapshots, int *nsnapshots)
+{
+ int ninitial = *nsnapshots;
+ char *out = NULL;
+ char *ret = control_send(&out, mero_host, mero_port, "", "snapshot
list", 1, mero_pass);
+ if (ret != NULL)
+ return ret;
+
+ if (strcmp(out, "OK1") == 0) {
+ // ok, empty resultset
+ free(out);
+ } else if (strncmp(out, "OK1\n", 4) == 0) {
+ // ok, nonempty resultset. Parse it.
+ char *p = out + 4;
+ char *end = p + strlen(p);
+ while (p < end) {
+ char *eol = strchr(p, '\n');
+ eol = (eol != NULL) ? eol : end;
+ int64_t time;
+ uint64_t size;
+ int len;
+ if (sscanf(p, "%" SCNd64 " %" SCNu64 " %n", &time,
&size, &len) != 2) {
+ free(out);
+ return strdup("internal parse error");
+ }
+ p += len;
+ char *dbend = strchr(p, ' ');
+ if (dbend == NULL) {
+ free(out);
+ return strdup("internal parse error");
+ }
+ int dblen = dbend - p;
+ char *path = dbend + 1;
+ int pathlen = eol - path;
+ struct snapshot *snap = push_snapshot(snapshots,
nsnapshots);
+ snap->dbname = malloc(dblen + 1);
+ memmove(snap->dbname, p, dblen);
+ snap->dbname[dblen] = '\0';
+ snap->time = time;
+ snap->size = size;
+ snap->path = malloc(pathlen + 1);
+ memmove(snap->path, path, pathlen);
+ p = eol + 1;
+ };
+ free(out);
+ } else {
+ return out;
+ }
+
+ // Sort them and give names of the form dbname@seqno
+ if (*nsnapshots > ninitial) {
+ int sort_len = *nsnapshots - ninitial;
+ struct snapshot *sort_start = *snapshots + ninitial;
+ qsort(sort_start, sort_len, sizeof(struct snapshot),
snapshot_enumerate_helper);
+ struct snapshot *prev = NULL;
+ int counter;
+ for (struct snapshot *cur = sort_start; cur < sort_start +
sort_len; cur++) {
+ if (prev == NULL || strcmp(prev->dbname, cur->dbname)
!= 0)
+ counter = 0;
+ counter++;
+ cur->name = malloc(strlen(cur->dbname) + 10);
+ sprintf(cur->name, "%s@%d", cur->dbname, counter);
+ prev = cur;
+ }
+ }
+
+ return NULL;
+}
+
+static void
+snapshot_list(int nglobs, char *globs[]) {
+ struct snapshot *snapshots = NULL;
+ int nsnapshots = 0;
+
+ // Retrieve the full snapshot list
+ char *err = snapshot_enumerate(&snapshots, &nsnapshots);
+ if (err != NULL) {
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
