Changeset: ce333a3dce58 for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/ce333a3dce58
Modified Files:
MonetDB.spec
gdk/gdk.h
gdk/gdk_bat.c
gdk/gdk_batop.c
gdk/gdk_bbp.c
gdk/gdk_tm.c
gdk/gdk_utils.c
monetdb5/mal/mal_interpreter.c
monetdb5/mal/mal_profiler.c
monetdb5/mal/mal_resource.c
monetdb5/modules/kernel/bat5.c
monetdb5/modules/mal/bbp.c
sql/backends/monet5/rel_bin.c
sql/backends/monet5/sql.c
sql/backends/monet5/sql_cat.c
sql/server/rel_optimize_sel.c
sql/server/rel_select.c
sql/server/rel_unnest.c
sql/server/sql_mvc.c
sql/server/sql_mvc.h
sql/server/sql_parser.y
sql/storage/bat/bat_storage.c
sql/storage/store.c
Branch: no_type_bat
Log Message:
merged with default
diffs (truncated from 6146 to 300 lines):
diff --git a/MonetDB.spec b/MonetDB.spec
--- a/MonetDB.spec
+++ b/MonetDB.spec
@@ -807,9 +807,7 @@ do
/usr/sbin/semodule -s ${selinuxvariant} -i \
%{_datadir}/selinux/${selinuxvariant}/monetdb.pp &> /dev/null || :
done
-# use /var/run/monetdb since that's what it says in the monetdb.fc file
-# it says that because /run/monetdb for some reason doesn't work
-/sbin/restorecon -R %{_localstatedir}/monetdb5 %{_localstatedir}/log/monetdb
/var/run/monetdb %{_bindir}/monetdbd %{_bindir}/mserver5
%{_unitdir}/monetdbd.service &> /dev/null || :
+/sbin/restorecon -R %{_localstatedir}/monetdb5 %{_localstatedir}/log/monetdb
%{_rundir}/monetdb %{_bindir}/monetdbd %{_bindir}/mserver5
%{_unitdir}/monetdbd.service &> /dev/null || :
/usr/bin/systemctl try-restart monetdbd.service
%postun selinux
@@ -838,6 +836,13 @@ fi
%setup -q
%build
+# from Fedora 40, selinux uses /run where before it used /var/run
+# the code is now for Fedora 40 but needs a patch for older versions
+%if (0%{?fedora} < 40)
+sed -i
's;@CMAKE_INSTALL_FULL_RUNSTATEDIR@/monetdb;@CMAKE_INSTALL_FULL_LOCALSTATEDIR@/run/monetdb;'
misc/selinux/monetdb.fc.in
+sed -i 's/1\.2/1.1/' misc/selinux/monetdb.te
+%endif
+
%cmake3 \
-DCMAKE_INSTALL_RUNSTATEDIR=/run \
-DRELEASE_VERSION=ON \
diff --git a/gdk/ChangeLog b/gdk/ChangeLog
--- a/gdk/ChangeLog
+++ b/gdk/ChangeLog
@@ -1,6 +1,14 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Tue Mar 26 2024 Sjoerd Mullender <[email protected]>
+- Made some changes to how BAT descriptors are allocated. They are now
+ allocated in bulk, meaning fewer malloc/free calls during processing.
+- Removed macro BBP_cache and its associated code. Checking whether a
+ BAT is cached (loaded in memory) can be done by checking the BBPLOADED
+ bit in the BBP_status value. Getting a pointer to the BAT descriptor
+ can be done by using BBP_desc.
+
* Tue Feb 6 2024 Sjoerd Mullender <[email protected]>
- The SQL transaction ID is no longer saved in the BBP.dir file.
diff --git a/gdk/ChangeLog.Dec2023 b/gdk/ChangeLog.Dec2023
--- a/gdk/ChangeLog.Dec2023
+++ b/gdk/ChangeLog.Dec2023
@@ -1,6 +1,12 @@
# ChangeLog file for GDK
# This file is updated with Maddlog
+* Thu Mar 28 2024 Sjoerd Mullender <[email protected]>
+- Threads have their own list of free bats. The list was not returned
+ to the system when a thread exited, meaning that the free bats that
+ were in the list would not be reused by any thread. This has been
+ fixed.
+
* Mon Mar 18 2024 Sjoerd Mullender <[email protected]>
- Fixed a couple of deadlock situations, one actually observed, one
never observed.
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -956,10 +956,9 @@ gdk_export void HEAPincref(Heap *h);
* field.
*/
typedef struct {
- BAT *cache; /* if loaded: BAT* handle */
char *logical; /* logical name (may point at bak) */
char bak[16]; /* logical name backup (tmp_%o) */
- BAT *desc; /* the BAT descriptor */
+ BAT descr; /* the BAT descriptor */
char *options; /* A string list of options */
#if SIZEOF_VOID_P == 4
char physical[20]; /* dir + basename for storage */
@@ -993,19 +992,18 @@ gdk_export BBPrec *BBP[N_BBPINIT];
/* fast defines without checks; internal use only */
#define BBP_record(i) BBP[(i)>>BBPINITLOG][(i)&(BBPINIT-1)]
-#define BBP_cache(i) BBP_record(i).cache
#define BBP_logical(i) BBP_record(i).logical
#define BBP_bak(i) BBP_record(i).bak
#define BBP_next(i) BBP_record(i).next
#define BBP_physical(i) BBP_record(i).physical
#define BBP_options(i) BBP_record(i).options
-#define BBP_desc(i) BBP_record(i).desc
+#define BBP_desc(i) (&BBP_record(i).descr)
#define BBP_refs(i) BBP_record(i).refs
#define BBP_lrefs(i) BBP_record(i).lrefs
#define BBP_status(i) ((unsigned) ATOMIC_GET(&BBP_record(i).status))
#define BBP_pid(i) BBP_record(i).pid
#define BATgetId(b) BBP_logical((b)->batCacheid)
-#define BBPvalid(i) (BBP_logical(i) != NULL && *BBP_logical(i) != '.')
+#define BBPvalid(i) (BBP_logical(i) != NULL)
#define BBPRENAME_ALREADY (-1)
#define BBPRENAME_ILLEGAL (-2)
diff --git a/gdk/gdk_aggr.c b/gdk/gdk_aggr.c
--- a/gdk/gdk_aggr.c
+++ b/gdk/gdk_aggr.c
@@ -3675,7 +3675,6 @@ BATmin_skipnil(BAT *b, void *aggr, bit s
if ((pb == NULL || bi.count == BATcount(pb)) &&
BATcheckimprints(b)) {
if (pb != NULL) {
- BAT *pb =
BBP_cache(VIEWtparent(b));
MT_lock_set(&pb->batIdxLock);
imprints = pb->timprints;
if (imprints != NULL)
@@ -3842,7 +3841,6 @@ BATmax_skipnil(BAT *b, void *aggr, bit s
if ((pb == NULL || BATcount(b) == BATcount(pb))
&&
BATcheckimprints(b)) {
if (pb != NULL) {
- BAT *pb =
BBP_cache(VIEWtparent(b));
MT_lock_set(&pb->batIdxLock);
imprints = pb->timprints;
if (imprints != NULL)
diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c
--- a/gdk/gdk_bat.c
+++ b/gdk/gdk_bat.c
@@ -58,22 +58,49 @@
BAT *
BATcreatedesc(oid hseq, int tt, bool heapnames, role_t role, uint16_t width)
{
+ bat bid;
BAT *bn;
+ Heap *h = NULL, *vh = NULL;
/*
* Alloc space for the BAT and its dependent records.
*/
assert(tt >= 0);
- bn = GDKmalloc(sizeof(BAT));
+ if (heapnames) {
+ if ((h = GDKmalloc(sizeof(Heap))) == NULL) {
+ return NULL;
+ }
+ *h = (Heap) {
+ .farmid = BBPselectfarm(role, tt, offheap),
+ .dirty = true,
+ };
- if (bn == NULL)
+ if (ATOMneedheap(tt)) {
+ if ((vh = GDKmalloc(sizeof(Heap))) == NULL) {
+ GDKfree(h);
+ return NULL;
+ }
+ *vh = (Heap) {
+ .farmid = BBPselectfarm(role, tt, varheap),
+ .dirty = true,
+ };
+ }
+ }
+
+ bid = BBPallocbat(tt);
+ if (bid == 0) {
+ GDKfree(h);
+ GDKfree(vh);
return NULL;
+ }
+ bn = BBP_desc(bid);
/*
* Fill in basic column info
*/
*bn = (BAT) {
+ .batCacheid = bid,
.hseqbase = hseq,
.ttype = tt,
@@ -90,39 +117,11 @@ BATcreatedesc(oid hseq, int tt, bool hea
.batRole = role,
.batTransient = true,
.batRestricted = BAT_WRITE,
+ .theap = h,
+ .tvheap = vh,
+ .creator_tid = MT_getpid(),
};
- if (heapnames) {
- if ((bn->theap = GDKmalloc(sizeof(Heap))) == NULL) {
- GDKfree(bn);
- return NULL;
- }
- *bn->theap = (Heap) {
- .farmid = BBPselectfarm(role, bn->ttype, offheap),
- .dirty = true,
- };
-
- if (ATOMneedheap(tt)) {
- if ((bn->tvheap = GDKmalloc(sizeof(Heap))) == NULL) {
- GDKfree(bn->theap);
- GDKfree(bn);
- return NULL;
- }
- *bn->tvheap = (Heap) {
- .farmid = BBPselectfarm(role, bn->ttype,
varheap),
- .dirty = true,
- };
- }
- }
- /*
- * add to BBP
- */
- if (BBPinsert(bn) == 0) {
- GDKfree(bn->tvheap);
- GDKfree(bn->theap);
- GDKfree(bn);
- return NULL;
- }
if (bn->theap) {
bn->theap->parentid = bn->batCacheid;
ATOMIC_INIT(&bn->theap->refs, 1);
@@ -727,7 +726,9 @@ BATdestroy(BAT *b)
HEAPdecref(b->oldtail, false);
b->oldtail = NULL;
}
- GDKfree(b);
+ *b = (BAT) {
+ .batCacheid = 0,
+ };
}
/*
@@ -2673,7 +2674,7 @@ BATassertProps(BAT *b)
assert(b != NULL);
assert(b->batCacheid > 0);
assert(b->batCacheid < getBBPsize());
- assert(b == BBP_cache(b->batCacheid));
+ assert(b == BBP_desc(b->batCacheid));
assert(b->batCount >= b->batInserted);
/* headless */
diff --git a/gdk/gdk_batop.c b/gdk/gdk_batop.c
--- a/gdk/gdk_batop.c
+++ b/gdk/gdk_batop.c
@@ -2149,7 +2149,7 @@ BATordered(BAT *b)
bat pbid = VIEWtparent(b);
MT_lock_unset(&b->theaplock);
if (pbid) {
- BAT *pb = BBP_cache(pbid);
+ BAT *pb = BBP_desc(pbid);
MT_lock_set(&pb->theaplock);
if (bi.count == BATcount(pb) &&
bi.h == pb->theap &&
@@ -2268,7 +2268,7 @@ BATordered_rev(BAT *b)
bat pbid = VIEWtparent(b);
MT_lock_unset(&b->theaplock);
if (pbid) {
- BAT *pb = BBP_cache(pbid);
+ BAT *pb = BBP_desc(pbid);
MT_lock_set(&pb->theaplock);
if (bi.count == BATcount(pb) &&
bi.h == pb->theap &&
diff --git a/gdk/gdk_bbp.c b/gdk/gdk_bbp.c
--- a/gdk/gdk_bbp.c
+++ b/gdk/gdk_bbp.c
@@ -101,8 +101,9 @@
* are found in O(1) by keeping a freelist that uses the 'next' field
* in the BBPrec records.
*/
-BBPrec *BBP[N_BBPINIT]; /* fixed base VM address of BBP array */
-bat BBPlimit = 0; /* current committed VM BBP array */
+static BBPrec BBP0[BBPINIT];
+BBPrec *BBP[N_BBPINIT] = {[0] = BBP0}; /* fixed base VM address of BBP array */
+bat BBPlimit = BBPINIT; /* current committed VM BBP array */
static ATOMIC_TYPE BBPsize = ATOMIC_VAR_INIT(0); /* current used size of BBP
array */
struct BBPfarm_t BBPfarms[MAXFARMS];
@@ -796,7 +797,8 @@ BBPreadEntries(FILE *fp, unsigned bbpver
}
ATOMIC_SET(&BBPsize, b.batCacheid + 1);
}
- if (BBP_desc(b.batCacheid) != NULL) {
+ BAT *bn = BBP_desc(b.batCacheid);
+ if (bn->batCacheid != 0) {
GDKfree(options);
TRC_CRITICAL(GDK, "duplicate entry in BBP.dir (ID = "
"%d) on line %d.", b.batCacheid, lineno);
@@ -816,11 +818,8 @@ BBPreadEntries(FILE *fp, unsigned bbpver
}
#endif
- BAT *bn;
Heap *hn;
- if ((bn = GDKmalloc(sizeof(BAT))) == NULL ||
- (hn = GDKmalloc(sizeof(Heap))) == NULL) {
- GDKfree(bn);
+ if ((hn = GDKmalloc(sizeof(Heap))) == NULL) {
GDKfree(options);
TRC_CRITICAL(GDK, "cannot allocate memory for BAT.");
goto bailout;
@@ -833,7 +832,6 @@ BBPreadEntries(FILE *fp, unsigned bbpver
assert(b.tvheap == &vh);
if ((vhn = GDKmalloc(sizeof(Heap))) == NULL) {
GDKfree(hn);
- GDKfree(bn);
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
