Changeset: e3cf471fb80e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/e3cf471fb80e
Modified Files:
gdk/gdk_logger.c
gdk/gdk_logger_internals.h
Branch: Jul2021
Log Message:
additional GDKerrors on missing bats
logger_activate now only triggers when we have enough 'bat' drops
new_logfile oly create a new file on enough drop or large enough log file
call new_logfile in log_tdone not (too early) in log_tend
diffs (166 lines):
diff --git a/gdk/gdk_logger.c b/gdk/gdk_logger.c
--- a/gdk/gdk_logger.c
+++ b/gdk/gdk_logger.c
@@ -274,6 +274,10 @@ la_bat_clear(logger *lg, logaction *la,
log_bid bid = internal_find_bat(lg, la->cid, tid);
BAT *b;
+ if (!bid) {
+ GDKerror("la_bat_clear failed to find bid for object %d\n",
la->cid);
+ return GDK_FAIL;
+ }
if (lg->debug & 1)
fprintf(stderr, "#la_bat_clear %d\n", la->cid);
@@ -639,8 +643,10 @@ la_bat_updates(logger *lg, logaction *la
log_bid bid = internal_find_bat(lg, la->cid, tid);
BAT *b = NULL;
- if (bid == 0)
- return GDK_SUCCEED; /* ignore bats no longer in the catalog */
+ if (!bid) {
+ GDKerror("la_bat_updates failed to find bid for object %d\n",
la->cid);
+ return GDK_FAIL;
+ }
if (!lg->flushing) {
b = BATdescriptor(bid);
@@ -742,6 +748,10 @@ la_bat_destroy(logger *lg, logaction *la
{
log_bid bid = internal_find_bat(lg, la->cid, tid);
+ if (!bid) {
+ GDKerror("la_bat_destroy failed to find bid for object %d\n",
la->cid);
+ return GDK_FAIL;
+ }
if (bid && logger_del_bat(lg, bid) != GDK_SUCCEED)
return GDK_FAIL;
return GDK_SUCCEED;
@@ -1031,6 +1041,7 @@ logger_open_output(logger *lg)
}
lg->end = 0;
+ lg->drops = 0;
if (!LOG_DISABLED(lg)) {
char id[32];
char *filename;
@@ -1629,8 +1640,6 @@ bm_subcommit(logger *lg)
sizes[i] = BATcount(dcatalog);
n[i++] = dcatalog->batCacheid;
- if (cleanup < (lg->cnt/2))
- cleanup = 0;
if (cleanup && (rcnt=cleanup_and_swap(lg, r, bids, lids, cnts,
catalog_bid, catalog_id, dcatalog, cleanup)) < 0) {
GDKfree(n);
GDKfree(r);
@@ -1649,7 +1658,7 @@ bm_subcommit(logger *lg)
sizes[i] = BATcount(lg->seqs_id);
n[i++] = lg->seqs_val->batCacheid;
}
- if (!cleanup && lg->seqs_id && BATcount(lg->dseqs) >
(BATcount(lg->seqs_id)/2)) {
+ if (!cleanup && lg->seqs_id && BATcount(lg->dseqs) >
(BATcount(lg->seqs_id)/2) && BATcount(lg->dseqs) > 10 ) {
BAT *tids, *ids, *vals;
tids = bm_tids(lg->seqs_id, lg->dseqs);
@@ -2105,6 +2114,8 @@ logger_new(int debug, const char *fn, co
.funcdata = funcdata,
.id = 0,
+ .drops = 0,
+ .end = 0,
.saved_id = getBBPlogno(), /* get saved log numer
from bbp */
.saved_tid = (int)getBBPtransid(), /* get saved
transaction id from bbp */
};
@@ -2240,14 +2251,15 @@ logger_cleanup_range(logger *lg, ulng id
gdk_return
logger_activate(logger *lg)
{
- if (lg->end > 0 && lg->saved_id+1 == lg->id) {
+ gdk_return res = GDK_SUCCEED;
+
+ if (lg->drops > 100000 && lg->end > 0 && lg->saved_id+1 == lg->id) {
lg->id++;
logger_close_output(lg);
/* start new file */
- if (logger_open_output(lg) != GDK_SUCCEED)
- return GDK_FAIL;
+ res = logger_open_output(lg);
}
- return GDK_SUCCEED;
+ return res;
}
gdk_return
@@ -2580,6 +2592,11 @@ log_bat_transient(logger *lg, log_id id)
log_bid bid = internal_find_bat(lg, id, -1);
logformat l;
+ if (!bid) {
+ GDKerror("log_bat_transient failed to find bid for object
%d\n", id);
+ return GDK_FAIL;
+ }
+
l.flag = LOG_DESTROY;
l.id = id;
@@ -2594,7 +2611,9 @@ log_bat_transient(logger *lg, log_id id)
if (lg->debug & 1)
fprintf(stderr, "#Logged destroyed bat (%d) %d\n", id,
bid);
- lg->end += BATcount(BBPquickdesc(bid));
+ BUN cnt = BATcount(BBPquickdesc(bid));
+ lg->end += cnt;
+ lg->drops += cnt;
gdk_return r = logger_del_bat(lg, bid);
logger_unlock(lg);
return r;
@@ -2718,7 +2737,7 @@ new_logfile(logger *lg)
p = (lng) getfilepos(getFile(lg->output_log));
if (p == -1)
return GDK_FAIL;
- if (p > log_large || (lg->end*1024) > log_large) {
+ if (lg->drops > 100000 || p > log_large || (lg->end*1024) > log_large) {
lg->id++;
logger_close_output(lg);
return logger_open_output(lg);
@@ -2748,8 +2767,7 @@ log_tend(logger *lg)
if (log_write_format(lg, &l) != GDK_SUCCEED ||
mnstr_flush(lg->output_log, MNSTR_FLUSH_DATA) ||
- (!(GDKdebug & NOSYNCMASK) && mnstr_fsync(lg->output_log)) ||
- new_logfile(lg) != GDK_SUCCEED) {
+ (!(GDKdebug & NOSYNCMASK) && mnstr_fsync(lg->output_log))) {
TRC_CRITICAL(GDK, "write failed\n");
return GDK_FAIL;
}
@@ -2766,6 +2784,8 @@ log_tdone(logger *lg, ulng commit_ts)
lg->current->last_tid = lg->tid;
lg->current->last_ts = commit_ts;
}
+ if (!LOG_DISABLED(lg) && new_logfile(lg) != GDK_SUCCEED)
+ GDKfatal("Could not create new log file\n");
return GDK_SUCCEED;
}
@@ -2928,6 +2948,10 @@ logger_find_bat(logger *lg, log_id id)
logger_lock(lg);
log_bid bid = internal_find_bat(lg, id, -1);
logger_unlock(lg);
+ if (!bid) {
+ GDKerror("logger_find_bat failed to find bid for object %d\n",
id);
+ return GDK_FAIL;
+ }
return bid;
}
diff --git a/gdk/gdk_logger_internals.h b/gdk/gdk_logger_internals.h
--- a/gdk/gdk_logger_internals.h
+++ b/gdk/gdk_logger_internals.h
@@ -26,6 +26,7 @@ struct logger {
int saved_tid; /* id of transaction which was flushed out
(into BBP storage) */
bool flushing;
bool flushnow;
+ ulng drops;
logged_range *pending;
logged_range *current;
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
