Changeset: 55a55f5fc953 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/55a55f5fc953
Modified Files:
clients/Tests/MAL-signatures.stable.out
clients/Tests/MAL-signatures.stable.out.int128
clients/Tests/exports.stable.out
gdk/gdk_logger.c
gdk/gdk_logger.h
monetdb5/optimizer/opt_emptybind.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql.h
sql/backends/monet5/sql_statement.c
sql/backends/monet5/vaults/fits/fits.c
sql/backends/monet5/vaults/netcdf/netcdf.c
sql/backends/monet5/vaults/shp/shp.c
sql/backends/monet5/wlr.c
sql/storage/bat/bat_logger.c
sql/storage/bat/bat_storage.c
sql/storage/bat/bat_table.c
sql/storage/sql_storage.h
sql/storage/store.c
tools/monetdbe/monetdbe.c
Branch: Jul2021
Log Message:
extended the claim and append api's too use offset BUN or offsets BAT. For
simple cases the offset is faster.
split log_tend into tend (flushing/syncing to disk unlocked) and tdone which
just marks the transaction as done in
the in memory structures of the logger.
cleanup (removed) msync calls where they became useless
small improvement of string_reader, ie use BUNappendmulti in chunks
diffs (truncated from 1174 to 300 lines):
diff --git a/clients/Tests/MAL-signatures.stable.out
b/clients/Tests/MAL-signatures.stable.out
--- a/clients/Tests/MAL-signatures.stable.out
+++ b/clients/Tests/MAL-signatures.stable.out
@@ -9208,7 +9208,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str):void ", "sql_analyze;", "" ]
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ]
[ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ",
"SQLany_cmp;", "" ]
-[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ]
+[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", ""
]
[ "sql", "argRecord", "pattern sql.argRecord():str ",
"SQLargRecord;", "" ]
[ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ",
"SQLargRecord;", "" ]
[ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ",
"SQLassert;", "" ]
@@ -9232,7 +9232,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", ""
]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1])
", "mvc_bind_idxbat_wrap;", "" ]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ",
"mvc_bind_idxbat_wrap;", "" ]
-[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ]
+[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", ""
]
[ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str,
X_1:str):lng ", "mvc_clear_table_wrap;", "" ]
[ "sql", "commit", "unsafe pattern sql.commit():void ",
"SQLcommit;", "" ]
[ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str,
X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str,
X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", ""
]
@@ -9685,7 +9685,7 @@ stdout of test 'MAL-signatures` in direc
[ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str,
X_1:str, X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str,
X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str,
X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ]
-[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:any...):int ", "WLRappend;", "" ]
+[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ]
[ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ",
"WLRcatalog;", "" ]
[ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int
", "WLRclear_table;", "" ]
[ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void
", "WLRgeneric;", "" ]
diff --git a/clients/Tests/MAL-signatures.stable.out.int128
b/clients/Tests/MAL-signatures.stable.out.int128
--- a/clients/Tests/MAL-signatures.stable.out.int128
+++ b/clients/Tests/MAL-signatures.stable.out.int128
@@ -12509,7 +12509,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str):void ", "sql_analyze;", "" ]
[ "sql", "analyze", "unsafe pattern sql.analyze(X_0:int, X_1:lng,
X_2:str, X_3:str, X_4:str):void ", "sql_analyze;", "" ]
[ "sql", "any", "pattern sql.any(X_0:bit, X_1:bit, X_2:bit):bit ",
"SQLany_cmp;", "" ]
-[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:bat[:oid], X_5:any):int ", "mvc_append_wrap;", "" ]
+[ "sql", "append", "pattern sql.append(X_0:int, X_1:str, X_2:str,
X_3:str, X_4:oid, X_5:bat[:oid], X_6:any):int ", "mvc_append_wrap;", ""
]
[ "sql", "argRecord", "pattern sql.argRecord():str ",
"SQLargRecord;", "" ]
[ "sql", "argRecord", "pattern sql.argRecord(X_0:any...):str ",
"SQLargRecord;", "" ]
[ "sql", "assert", "pattern sql.assert(X_0:bit, X_1:str):void ",
"SQLassert;", "" ]
@@ -12535,7 +12535,7 @@ stdout of test 'MAL-signatures` in direc
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int):bat[:any_1] ", "mvc_bind_idxbat_wrap;", ""
]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int) (X_7:bat[:oid], X_8:bat[:any_1])
", "mvc_bind_idxbat_wrap;", "" ]
[ "sql", "bind_idxbat", "pattern sql.bind_idxbat(X_0:int, X_1:str,
X_2:str, X_3:str, X_4:int, X_5:int, X_6:int):bat[:any_1] ",
"mvc_bind_idxbat_wrap;", "" ]
-[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng):bat[:oid] ", "mvc_claim_wrap;", "" ]
+[ "sql", "claim", "unsafe pattern sql.claim(X_0:int, X_1:str,
X_2:str, X_3:lng) (X_4:oid, X_5:bat[:oid]) ", "mvc_claim_wrap;", ""
]
[ "sql", "clear_table", "unsafe pattern sql.clear_table(X_0:str,
X_1:str):lng ", "mvc_clear_table_wrap;", "" ]
[ "sql", "commit", "unsafe pattern sql.commit():void ",
"SQLcommit;", "" ]
[ "sql", "copy_from", "unsafe pattern sql.copy_from(X_0:ptr, X_1:str,
X_2:str, X_3:str, X_4:str, X_5:str, X_6:lng, X_7:lng, X_8:int, X_9:str,
X_10:int, X_11:int):bat[:any]... ", "mvc_import_table_wrap;", ""
]
@@ -13008,7 +13008,7 @@ stdout of test 'MAL-signatures` in direc
[ "wlr", "alter_set_table", "pattern wlr.alter_set_table(X_0:str,
X_1:str, X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_table", "pattern wlr.alter_table(X_0:str, X_1:str,
X_2:int):void ", "WLRgeneric;", "" ]
[ "wlr", "alter_user", "pattern wlr.alter_user(X_0:str, X_1:str,
X_2:int, X_3:str, X_4:str):void ", "WLRgeneric;", "" ]
-[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:any...):int ", "WLRappend;", "" ]
+[ "wlr", "append", "pattern wlr.append(X_0:str, X_1:str, X_2:str,
X_3:oid, X_4:bat[:oid], X_5:any...):int ", "WLRappend;", "" ]
[ "wlr", "catalog", "pattern wlr.catalog(X_0:str):void ",
"WLRcatalog;", "" ]
[ "wlr", "clear_table", "pattern wlr.clear_table(X_0:str, X_1:str):int
", "WLRclear_table;", "" ]
[ "wlr", "comment_on", "pattern wlr.comment_on(X_0:int, X_1:str):void
", "WLRgeneric;", "" ]
diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out
--- a/clients/Tests/exports.stable.out
+++ b/clients/Tests/exports.stable.out
@@ -541,7 +541,8 @@ gdk_return log_bat_transient(logger *lg,
gdk_return log_constant(logger *lg, int type, ptr val, log_id id, lng offset,
lng cnt);
gdk_return log_delta(logger *lg, BAT *uid, BAT *uval, log_id id);
gdk_return log_sequence(logger *lg, int seq, lng id);
-gdk_return log_tend(logger *lg, ulng commit_ts);
+gdk_return log_tdone(logger *lg, ulng commit_ts);
+gdk_return log_tend(logger *lg);
gdk_return log_tstart(logger *lg, bool flush);
gdk_return logger_activate(logger *lg);
lng logger_changes(logger *lg);
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -347,14 +347,24 @@ string_reader(logger *lg, BAT *b, lng nr
/* handle strings */
if (b) {
char *t = buf;
+ /* chunked */
+#define CHUNK_SIZE 1024
+ char *strings[CHUNK_SIZE];
+ int cur = 0;
+
for(int i=0; i<nr && res == LOG_OK; i++) {
- if (BUNappend(b, t, true) != GDK_SUCCEED)
+ strings[cur++] = t;
+ if (cur == CHUNK_SIZE && BUNappendmulti(b, strings,
cur, true) != GDK_SUCCEED)
res = LOG_ERR;
+ if (cur == CHUNK_SIZE)
+ cur = 0;
/* find next */
while(*t)
t++;
t++;
}
+ if (cur && BUNappendmulti(b, strings, cur, true) != GDK_SUCCEED)
+ res = LOG_ERR;
}
GDKfree(buf);
return res;
@@ -2641,7 +2651,7 @@ new_logfile(logger *lg)
}
gdk_return
-log_tend(logger *lg, ulng commit_ts)
+log_tend(logger *lg)
{
logformat l;
gdk_return res = GDK_SUCCEED;
@@ -2657,11 +2667,6 @@ log_tend(logger *lg, ulng commit_ts)
return res;
}
- if (lg->current) {
- lg->current->last_tid = lg->tid;
- lg->current->last_ts = commit_ts;
- }
-
if (LOG_DISABLED(lg)) {
lg->end++;
return GDK_SUCCEED;
@@ -2678,6 +2683,19 @@ log_tend(logger *lg, ulng commit_ts)
return GDK_SUCCEED;
}
+gdk_return
+log_tdone(logger *lg, ulng commit_ts)
+{
+ if (lg->debug & 1)
+ fprintf(stderr, "#log_tdone %d\n", lg->tid);
+
+ if (lg->current) {
+ lg->current->last_tid = lg->tid;
+ lg->current->last_ts = commit_ts;
+ }
+ return GDK_SUCCEED;
+}
+
static gdk_return
log_sequence_(logger *lg, int seq, lng val, int flush)
{
diff --git a/gdk/gdk_logger.h b/gdk/gdk_logger.h
--- a/gdk/gdk_logger.h
+++ b/gdk/gdk_logger.h
@@ -69,7 +69,8 @@ gdk_export gdk_return log_delta(logger *
//gdk_export gdk_return log_batgroup_end(logger *lg, oid id);
gdk_export gdk_return log_tstart(logger *lg, bool flush);
-gdk_export gdk_return log_tend(logger *lg, ulng commit_ts);
+gdk_export gdk_return log_tend(logger *lg);
+gdk_export gdk_return log_tdone(logger *lg, ulng commit_ts);
gdk_export gdk_return log_sequence(logger *lg, int seq, lng id);
gdk_export log_bid logger_find_bat(logger *lg, log_id id);
diff --git a/monetdb5/optimizer/opt_emptybind.c
b/monetdb5/optimizer/opt_emptybind.c
--- a/monetdb5/optimizer/opt_emptybind.c
+++ b/monetdb5/optimizer/opt_emptybind.c
@@ -133,8 +133,9 @@ OPTemptybindImplementation(Client cntxt,
for(j= 0; j< etop; j++){
q= updated[j];
if(q && getModuleId(q) == sqlRef &&
isUpdateInstruction(q)){
- if (
strcmp(getVarConstant(mb,getArg(q,2)).val.sval, sch) == 0 &&
-
strcmp(getVarConstant(mb,getArg(q,3)).val.sval, tbl) == 0 ){
+ int c = getFunctionId(q) == claimRef;
/* claim has 2 results */
+ if (
strcmp(getVarConstant(mb,getArg(q,2+c)).val.sval, sch) == 0 &&
+
strcmp(getVarConstant(mb,getArg(q,3+c)).val.sval, tbl) == 0 ){
empty[getArg(p,0)] = 0;
if( p->retc == 2){
empty[getArg(p,1)] = 0;
diff --git a/sql/backends/monet5/sql.c b/sql/backends/monet5/sql.c
--- a/sql/backends/monet5/sql.c
+++ b/sql/backends/monet5/sql.c
@@ -491,22 +491,24 @@ create_table_or_view(mvc *sql, char* sna
return MAL_SUCCEED;
}
-static BAT *
-mvc_claim_slots(sql_trans *tr, sql_table *t, size_t cnt)
+static int
+mvc_claim_slots(sql_trans *tr, sql_table *t, size_t cnt, BUN *offset, BAT
**pos)
{
sqlstore *store = tr->store;
- return store->storage_api.claim_tab(tr, t, cnt);
+ return store->storage_api.claim_tab(tr, t, cnt, offset, pos);
}
str
mvc_claim_wrap(Client cntxt, MalBlkPtr mb, MalStkPtr stk, InstrPtr pci)
{
- bat *res = getArgReference_bat(stk, pci, 0);
+ BUN *offset = (BUN*)getArgReference_oid(stk, pci, 0);
+ bat *res = getArgReference_bat(stk, pci, 1);
mvc *m = NULL;
str msg;
- const char *sname = *getArgReference_str(stk, pci, 2);
- const char *tname = *getArgReference_str(stk, pci, 3);
- lng cnt = *(lng*)getArgReference_lng(stk, pci, 4);
+ const char *sname = *getArgReference_str(stk, pci, 3);
+ const char *tname = *getArgReference_str(stk, pci, 4);
+ lng cnt = *(lng*)getArgReference_lng(stk, pci, 5);
+ BAT *pos = NULL;
sql_schema *s;
sql_table *t;
@@ -523,9 +525,10 @@ mvc_claim_wrap(Client cntxt, MalBlkPtr m
t = mvc_bind_table(m, s, tname);
if (t == NULL)
throw(SQL, "sql.claim", SQLSTATE(42S02) "Table missing %s.%s",
sname, tname);
- BAT *pos = mvc_claim_slots(m->session->tr, t, (size_t)cnt);
- if (pos) {
- BBPkeepref(*res = pos->batCacheid);
+ if (mvc_claim_slots(m->session->tr, t, (size_t)cnt, offset, &pos) ==
LOG_OK) {
+ *res = bat_nil;
+ if (pos)
+ BBPkeepref(*res = pos->batCacheid);
return MAL_SUCCEED;
}
throw(SQL, "sql.claim", SQLSTATE(3F000) "Could not claim slots");
@@ -578,8 +581,9 @@ create_table_from_emit(Client cntxt, cha
return msg;
if (!(t = mvc_bind_table(sql, s, tname)))
return sql_error(sql, ERR_NOTFOUND, SQLSTATE(3F000) "CREATE
TABLE: could not bind table %s", tname);
- BAT *pos = mvc_claim_slots(sql->session->tr, t, BATcount(columns[0].b));
- if (!pos)
+ BUN offset;
+ BAT *pos = NULL;
+ if (mvc_claim_slots(sql->session->tr, t, BATcount(columns[0].b),
&offset, &pos) != LOG_OK)
return sql_error(sql, 02, SQLSTATE(3F000) "CREATE TABLE: Could
not insert data");
for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
@@ -589,7 +593,7 @@ create_table_from_emit(Client cntxt, cha
bat_destroy(pos);
return sql_error(sql, ERR_NOTFOUND, SQLSTATE(3F000)
"CREATE TABLE: could not bind column %s", columns[i].name);
}
- if ((msg = mvc_append_column(sql->session->tr, col, pos, b)) !=
MAL_SUCCEED) {
+ if ((msg = mvc_append_column(sql->session->tr, col, offset,
pos, b)) != MAL_SUCCEED) {
bat_destroy(pos);
return msg;
}
@@ -618,8 +622,9 @@ append_to_table_from_emit(Client cntxt,
return sql_error(sql, ERR_NOTFOUND, SQLSTATE(3F000) "APPEND
TABLE: no such schema '%s'", sname);
if (!(t = mvc_bind_table(sql, s, tname)))
return sql_error(sql, ERR_NOTFOUND, SQLSTATE(3F000) "APPEND
TABLE: could not bind table %s", tname);
- BAT *pos = mvc_claim_slots(sql->session->tr, t, BATcount(columns[0].b));
- if (!pos)
+ BUN offset;
+ BAT *pos = NULL;
+ if (mvc_claim_slots(sql->session->tr, t, BATcount(columns[0].b),
&offset, &pos) != LOG_OK)
return sql_error(sql, 02, SQLSTATE(3F000) "APPEND TABLE: Could
not append data");
for (i = 0; i < ncols; i++) {
BAT *b = columns[i].b;
@@ -629,7 +634,7 @@ append_to_table_from_emit(Client cntxt,
bat_destroy(pos);
return sql_error(sql, ERR_NOTFOUND, SQLSTATE(3F000)
"APPEND TABLE: could not bind column %s", columns[i].name);
}
- if ((msg = mvc_append_column(sql->session->tr, col, pos, b)) !=
MAL_SUCCEED) {
+ if ((msg = mvc_append_column(sql->session->tr, col, offset,
pos, b)) != MAL_SUCCEED) {
bat_destroy(pos);
return msg;
}
@@ -1753,10 +1758,10 @@ mvc_bind_idxbat_wrap(Client cntxt, MalBl
}
str
-mvc_append_column(sql_trans *t, sql_column *c, BAT *pos, BAT *ins)
+mvc_append_column(sql_trans *t, sql_column *c, BUN offset, BAT *pos, BAT *ins)
{
sqlstore *store = t->store;
- int res = store->storage_api.append_col(t, c, pos, ins, TYPE_bat);
+ int res = store->storage_api.append_col(t, c, offset, pos, ins,
BATcount(ins), TYPE_bat);
if (res != LOG_OK) /* the conflict case should never happen, but leave
it here */
throw(SQL, "sql.append", SQLSTATE(42000) "Append failed%s", res
== LOG_CONFLICT ? " due to conflict with another transaction" : "");
return MAL_SUCCEED;
@@ -1812,14 +1817,16 @@ mvc_append_wrap(Client cntxt, MalBlkPtr
const char *sname = *getArgReference_str(stk, pci, 2);
const char *tname = *getArgReference_str(stk, pci, 3);
const char *cname = *getArgReference_str(stk, pci, 4);
- bat Pos = *getArgReference_bat(stk, pci, 5);
- ptr ins = getArgReference(stk, pci, 6);
- int tpe = getArgType(mb, pci, 6), log_res = LOG_OK;
+ BUN offset = *(BUN*)getArgReference_oid(stk, pci, 5);
+ bat Pos = *getArgReference_bat(stk, pci, 6);
+ ptr ins = getArgReference(stk, pci, 7);
+ int tpe = getArgType(mb, pci, 7), log_res = LOG_OK;
sql_schema *s;
sql_table *t;
sql_column *c;
sql_idx *i;
BAT *b = NULL, *pos = NULL;
+ BUN cnt = 1;
*res = 0;
if ((msg = getSQLContext(cntxt, mb, &m, NULL)) != NULL)
@@ -1828,7 +1835,7 @@ mvc_append_wrap(Client cntxt, MalBlkPtr
return msg;
if (tpe > GDKatomcnt)
tpe = TYPE_bat;
- if ((pos = BATdescriptor(Pos)) == NULL)
_______________________________________________
checkin-list mailing list
[email protected]
https://www.monetdb.org/mailman/listinfo/checkin-list
