Changeset: 1e6719365d2b for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/1e6719365d2b
Added Files:
sql/scripts/77_storage.sql
Modified Files:
sql/backends/monet5/CMakeLists.txt
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/scripts/58_hot_snapshot.sql
sql/scripts/CMakeLists.txt
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_storage.h
sql/storage/sql_storage.h
sql/storage/store.c
Branch: insertonly
Log Message:
Set sql_debug=4 so deltas in append only tables are not logged into WAL.
New function to explicitly persist deltas in append only tables.
diffs (truncated from 358 to 300 lines):
diff --git a/sql/backends/monet5/CMakeLists.txt
b/sql/backends/monet5/CMakeLists.txt
--- a/sql/backends/monet5/CMakeLists.txt
+++ b/sql/backends/monet5/CMakeLists.txt
@@ -113,6 +113,7 @@ set(include_sql_files
58_hot_snapshot
75_storagemodel
76_dump
+ 77_storage
80_statistics
81_tracer
91_information_schema)
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -4310,6 +4310,164 @@ end:
}
str
+SQLinsertonly_persist(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
+{
+ (void)stk;
+ (void)pci;
+
+ bat *r0 = getArgReference_bat(stk,pci,0),
+ *r1 = getArgReference_bat(stk,pci,1),
+ *r2 = getArgReference_bat(stk,pci,2);
+
+ str msg = MAL_SUCCEED;
+ sqlstore *store = NULL;
+ mvc *m = NULL;
+ sql_trans *tr = NULL;
+ struct os_iter si = {0};
+ node *ncol;
+ storage *t_storage = NULL;
+
+ BAT *bs = NULL, *tables = NULL, *tables_ids = NULL, *rowcounts = NULL;
+
+ int n = 1000;
+ bat *commit_list = GDKmalloc(sizeof(bat) * (n + 1));
+ BUN *sizes = GDKmalloc(sizeof(BUN) * (n + 1));
+
+ commit_list[0] = 0;
+ sizes[0] = 0;
+ int i = 1;
+
+ msg = getSQLContext(cntxt, mb, &m, NULL);
+
+ if (msg)
+ return msg;
+
+ store = m->session->tr->store;
+ tr = m->session->tr;
+
+ tables = COLnew(0, TYPE_str, 0, TRANSIENT);
+ tables_ids = COLnew(0, TYPE_int, 0, TRANSIENT);
+ rowcounts = COLnew(0, TYPE_lng, 0, TRANSIENT);
+
+ if (!store->skip_insertonly) /* nothing needed, return empty results */
+ goto exit2;
+
+ if (tables == NULL || tables_ids == NULL || rowcounts == NULL ||
commit_list == NULL || sizes == NULL) {
+ msg = createException(SQL, "sql.insertonly_persist",
SQLSTATE(HY001));
+ goto exit2;
+ }
+
+ MT_lock_set(&store->commit);
+
+ os_iterator(&si, tr->cat->schemas, tr, NULL);
+
+ for (sql_base *b = oi_next(&si); b; b = oi_next(&si)) {
+ sql_schema *s = (sql_schema *) b;
+ str x = s->base.name;
+
+ if (strcmp(x, "sys") == 0 || strcmp(x, "tmp") == 0 ||
+ strcmp(x, "json") == 0 || strcmp(x, "profiler") == 0 ||
+ strcmp(x, "logging") == 0)
+ continue;
+
+ if (s->tables) {
+ struct os_iter oi;
+ os_iterator(&oi, s->tables, tr, NULL);
+
+ for (sql_base *bt = oi_next(&oi); bt; bt =
oi_next(&oi)) {
+ sql_table *t = (sql_table *) bt;
+
+ if (isTable(t) && t->access ==
TABLE_APPENDONLY) {
+ str t_name = t->base.name;
+ sqlid t_id = t->base.id;
+ t_storage = bind_del_data(tr, t, NULL);
+
+ if (ol_first_node(t->columns)) {
+
+ for (ncol =
ol_first_node((t)->columns); ncol; ncol = ncol->next) {
+ sql_column *c =
(sql_column *) ncol->data;
+ bs =
store->storage_api.bind_col(tr, c, RDONLY);
+
+ if (bs == NULL) {
+ msg =
createException(SQL, "insertonly_persist",
+
SQLSTATE(HY005) "Cannot access column descriptor");
+ goto exit1;
+ }
+
+ if (isVIEW(bs)) {
+ bs =
BATdescriptor(VIEWtparent(bs));
+ if (bs == NULL)
{
+ msg =
createException(SQL, "insertonly_persist",
+
SQLSTATE(HY005) "Cannot access column
descriptor");
+ goto
exit1;
+ }
+ }
+
+ if (i == n &&
ncol->next) {
+ n = n * 2;
+ commit_list =
GDKrealloc(commit_list, sizeof(bat) * n);
+ sizes =
GDKrealloc(sizes, sizeof(BUN) * n);
+ }
+
+ if (commit_list == NULL
|| sizes == NULL) {
+ msg =
createException(SQL, "insertonly_persist", SQLSTATE(HY001));
+ goto exit2;
+ }
+
+ commit_list[i] =
t_storage->cs.bid;
+ commit_list[i+1] =
bs->theap->parentid;
+ sizes[i] =
!bs->batTransient ? BATcount(bs) : 0;
+ sizes[i+1] =
!bs->batTransient ? BATcount(bs) : 0;
+
+ if (BUNappend(tables,
t_name, false) != GDK_SUCCEED) {
+ msg =
createException(SQL, "insertonly_persist", "Failed to append 'table'");
+ goto exit1;
+ }
+
+ if
(BUNappend(tables_ids, &t_id, false) != GDK_SUCCEED) {
+ msg =
createException(SQL, "insertonly_persist", "Failed to append 'table_id'");
+ goto exit1;
+ }
+
+ /* if
(BUNappend(rowcounts, &(lng){sizes[i]}, false) != GDK_SUCCEED) { */
+ if
(BUNappend(rowcounts, sizes + i, false) != GDK_SUCCEED) {
+ msg =
createException(SQL, "insertonly_persist", "Failed to append 'rowcount'");
+ goto exit1;
+ }
+
+ i+=2;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (TMsubcommit_list(commit_list, sizes, i, -1, -1) != GDK_SUCCEED)
+ msg = createException(SQL, "insertonly_persist", GDK_EXCEPTION);
+
+ exit1:
+ MT_lock_unset(&store->commit);
+
+ exit2:
+ GDKfree(commit_list);
+ GDKfree(sizes);
+ if (msg) {
+ BBPreclaim(tables);
+ BBPreclaim(tables_ids);
+ BBPreclaim(rowcounts);
+ } else {
+ *r0 = tables->batCacheid;
+ *r1 = tables_ids->batCacheid;
+ *r2 = rowcounts->batCacheid;
+ BBPkeepref(tables);
+ BBPkeepref(tables_ids);
+ BBPkeepref(rowcounts);
+ }
+ return msg;
+}
+
+str
SQLsession_prepared_statements(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci)
{
BAT *sessionid, *user, *statementid, *statement, *created;
@@ -5061,6 +5219,7 @@ static mel_func sql_init_funcs[] = {
pattern("sql", "resume_log_flushing", SQLresume_log_flushing, true, "Resume
WAL log flushing", args(1,1, arg("",void))),
pattern("sql", "suspend_log_flushing", SQLsuspend_log_flushing, true,
"Suspend WAL log flushing", args(1,1, arg("",void))),
pattern("sql", "hot_snapshot", SQLhot_snapshot, true, "Write db snapshot to
the given tar(.gz/.lz4/.bz/.xz) file on either server or client", args(1,3,
arg("",void),arg("tarfile", str),arg("onserver",bit))),
+ pattern("sql", "insertonly_persist", SQLinsertonly_persist, true, "Persist
changes to new data on append only tables.", args(3, 3, batarg("table", str),
batarg("table_id", int), batarg("rowcount", lng))),
pattern("sql", "assert", SQLassert, false, "Generate an exception when
b==true", args(1,3, arg("",void),arg("b",bit),arg("msg",str))),
pattern("sql", "assert", SQLassertInt, false, "Generate an exception when
b!=0", args(1,3, arg("",void),arg("b",int),arg("msg",str))),
pattern("sql", "assert", SQLassertLng, false, "Generate an exception when
b!=0", args(1,3, arg("",void),arg("b",lng),arg("msg",str))),
diff --git a/sql/backends/monet5/sql.h b/sql/backends/monet5/sql.h
--- a/sql/backends/monet5/sql.h
+++ b/sql/backends/monet5/sql.h
@@ -283,6 +283,7 @@ extern str SQLsuspend_log_flushing(Clien
extern str SQLresume_log_flushing(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
extern str SQLhot_snapshot(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr
pci);
extern str SQLhot_snapshot_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
+extern str SQLinsertonly_persist(Client cntxt, MalBlkPtr mb, MalStkPtr stk,
InstrPtr pci);
extern str SQLsession_prepared_statements(Client cntxt, MalBlkPtr mb,
MalStkPtr stk, InstrPtr pci);
extern str SQLsession_prepared_statements_args(Client cntxt, MalBlkPtr mb,
MalStkPtr stk, InstrPtr pci);
diff --git a/sql/scripts/58_hot_snapshot.sql b/sql/scripts/58_hot_snapshot.sql
--- a/sql/scripts/58_hot_snapshot.sql
+++ b/sql/scripts/58_hot_snapshot.sql
@@ -27,4 +27,3 @@ create user ".snapshot"
grant execute on procedure sys.hot_snapshot(string) to ".snapshot";
grant execute on procedure sys.hot_snapshot(string, bool) to ".snapshot";
-
diff --git a/sql/scripts/77_storage.sql b/sql/scripts/77_storage.sql
new file mode 100644
--- /dev/null
+++ b/sql/scripts/77_storage.sql
@@ -0,0 +1,16 @@
+-- SPDX-License-Identifier: MPL-2.0
+--
+-- This Source Code Form is subject to the terms of the Mozilla Public
+-- License, v. 2.0. If a copy of the MPL was not distributed with this
+-- file, You can obtain one at http://mozilla.org/MPL/2.0/.
+--
+-- Copyright 1997 - July 2008 CWI, August 2008 - 2023 MonetDB B.V.
+
+create function sys.insertonly_persist()
+returns table(
+ "table" string,
+ "table_id" int,
+ "rowcount" bigint
+)
+external name sql.insertonly_persist;
+grant execute on function sys.insertonly_persist() to public;
diff --git a/sql/scripts/CMakeLists.txt b/sql/scripts/CMakeLists.txt
--- a/sql/scripts/CMakeLists.txt
+++ b/sql/scripts/CMakeLists.txt
@@ -36,6 +36,7 @@ install(FILES
58_hot_snapshot.sql
75_storagemodel.sql
76_dump.sql
+ 77_storage.sql
80_statistics.sql
81_tracer.sql
91_information_schema.sql
diff --git a/sql/storage/bat/bat_storage.c b/sql/storage/bat/bat_storage.c
--- a/sql/storage/bat/bat_storage.c
+++ b/sql/storage/bat/bat_storage.c
@@ -2536,7 +2536,7 @@ segments_conflict(sql_trans *tr, segment
static int clear_storage(sql_trans *tr, sql_table *t, storage *s);
-static storage *
+storage *
bind_del_data(sql_trans *tr, sql_table *t, bool *clear)
{
storage *obat;
@@ -3868,6 +3868,9 @@ log_table_append(sql_trans *tr, sql_tabl
size_t end = segs_end(segs, tr, t);
+ if (store->skip_insertonly && t->access == TABLE_APPENDONLY)
+ return LOG_OK;
+
if (tr_log_table_start(tr, t) != LOG_OK)
return LOG_ERR;
@@ -4060,11 +4063,18 @@ savepoint_commit_delta( sql_delta *delta
static int
log_update_col( sql_trans *tr, sql_change *change)
{
+ sqlstore *store = tr->store;
sql_column *c = (sql_column*)change->obj;
assert(!isTempTable(c->t));
- if (isDeleted(c->t))
+ if (isDeleted(c->t)) {
change->handled = true;
+ return LOG_OK;
+ }
+
+ if ((store->skip_insertonly && c->t->access == TABLE_APPENDONLY))
+ return LOG_OK;
+
if (!isDeleted(c->t) && !tr->parent) {/* don't write save point commits
*/
storage *s = ATOMIC_PTR_GET(&c->t->data);
sql_delta *d = ATOMIC_PTR_GET(&c->data);
@@ -4171,11 +4181,15 @@ commit_update_col( sql_trans *tr, sql_ch
static int
log_update_idx( sql_trans *tr, sql_change *change)
{
+ sqlstore *store = tr->store;
sql_idx *i = (sql_idx*)change->obj;
assert(!isTempTable(i->t));
- if (isDeleted(i->t))
+ if (isDeleted(i->t) || (store->skip_insertonly && i->t->access ==
TABLE_APPENDONLY)) {
change->handled = true;
+ return LOG_OK;
+ }
+
if (!isDeleted(i->t) && !tr->parent) { /* don't write save point
commits */
storage *s = ATOMIC_PTR_GET(&i->t->data);
sql_delta *d = ATOMIC_PTR_GET(&i->data);
@@ -4221,11 +4235,18 @@ savepoint_commit_storage( storage *dbat,
static int
log_update_del( sql_trans *tr, sql_change *change)
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
