Changeset: c21d9793079e for MonetDB
URL: https://dev.monetdb.org/hg/MonetDB/rev/c21d9793079e
Modified Files:
gdk/gdk.h
gdk/gdk_heap.c
gdk/gdk_utils.c
Branch: async-heap-remove
Log Message:
Add async heap file remove
Currently since the heap files seem to be reused, this solution might free
files currently being used
diffs (124 lines):
diff --git a/gdk/gdk.h b/gdk/gdk.h
--- a/gdk/gdk.h
+++ b/gdk/gdk.h
@@ -576,6 +576,17 @@ typedef struct {
bat parentid; /* cache id of VIEW parent bat */
} Heap;
+/*
+ * Instead of removing the heap files straight away, which incurs some impact
on response time,
+ * we add the path to a buffer and asynchronously remove them.
+ */
+extern char **HEAPfreeBuffer; /* stores the heap filenames to remove */
+extern MT_Lock HEAPfreeBufferLock; /* controls concurrent accesses to the heap
free buffer */
+extern int HEAPfreeBufferCount; /* current number of files stored in the heap
free buffer */
+extern void HEAPfreeWorker(void*);
+#define HEAP_FREE_BUFFER_SIZE 1024 /* maximum heap free buffer size */
+#define HEAP_FREE_WORKER_DELTA_MS 1024 /* sleep time between worker executions
(in milliseconds) */
+
typedef struct Hash Hash;
typedef struct Imprints Imprints;
typedef struct Strimps Strimps;
diff --git a/gdk/gdk_heap.c b/gdk/gdk_heap.c
--- a/gdk/gdk_heap.c
+++ b/gdk/gdk_heap.c
@@ -568,6 +568,49 @@ HEAPcopy(Heap *dst, Heap *src, size_t of
return GDK_FAIL;
}
+char **HEAPfreeBuffer;
+MT_Lock HEAPfreeBufferLock;
+int HEAPfreeBufferCount = 0;
+
+/*
+ * Asynchronously frees heap files
+ */
+__attribute__((__noreturn__))
+void
+HEAPfreeWorker(void* _) {
+ (void)_;
+ while (true) {
+ /* make a copy of the buffer and clean it to avoid blocking
transactions for too long */
+ MT_lock_set(&HEAPfreeBufferLock);
+ char** bufferCopy = malloc(HEAPfreeBufferCount * sizeof(char*));
+ int count = HEAPfreeBufferCount;
+ for (int i = 0; i < count; i++) {
+ bufferCopy[i] = HEAPfreeBuffer[i];
+ }
+ HEAPfreeBufferCount = 0; /* clean the buffer */
+ MT_lock_unset(&HEAPfreeBufferLock);
+
+ /* free the heap files */
+ for (int i = 0; i < count; i++) {
+ char *path = bufferCopy[i];
+ if (path && MT_remove(path) != 0 && errno != ENOENT) {
+ perror(path);
+ }
+ char *pathNew = malloc((strlen(path) + 4) *
sizeof(char));
+ snprintf(pathNew, sizeof(*pathNew), "%s.new", path);
+ if (MT_remove(pathNew) != 0 && errno != ENOENT) {
+ perror(pathNew);
+ }
+ GDKfree(path);
+ free(pathNew);
+ }
+
+ free(bufferCopy);
+
+ MT_sleep_ms(HEAP_FREE_WORKER_DELTA_MS);
+ }
+}
+
/* Free the memory associated with the heap H.
* Unlinks (removes) the associated file if the rmheap flag is set. */
void
@@ -604,13 +647,25 @@ HEAPfree(Heap *h, bool rmheap)
#endif
if (rmheap && !GDKinmemory(h->farmid)) {
char *path = GDKfilepath(h->farmid, BATDIR, h->filename, NULL);
- if (path && MT_remove(path) != 0 && errno != ENOENT)
- perror(path);
- GDKfree(path);
- path = GDKfilepath(h->farmid, BATDIR, h->filename, "new");
- if (path && MT_remove(path) != 0 && errno != ENOENT)
- perror(path);
- GDKfree(path);
+ MT_lock_set(&HEAPfreeBufferLock);
+ /* check if the buffer has space */
+ if (HEAPfreeBufferCount < HEAP_FREE_BUFFER_SIZE) {
+ HEAPfreeBuffer[HEAPfreeBufferCount++] = path;
+ MT_lock_unset(&HEAPfreeBufferLock);
+ }
+ /* in case there is no space on the buffer, free the heap
synchronously */
+ else {
+ MT_lock_unset(&HEAPfreeBufferLock);
+ if (path && MT_remove(path) != 0 && errno != ENOENT) {
+ perror(path);
+ }
+ GDKfree(path);
+ path = GDKfilepath(h->farmid, BATDIR, h->filename,
"new");
+ if (path && MT_remove(path) != 0 && errno != ENOENT) {
+ perror(path);
+ }
+ GDKfree(path);
+ }
}
}
diff --git a/gdk/gdk_utils.c b/gdk/gdk_utils.c
--- a/gdk/gdk_utils.c
+++ b/gdk/gdk_utils.c
@@ -893,6 +893,13 @@ GDKinit(opt *set, int setlen, bool embed
int i, nlen = 0;
char buf[16];
+ /* heapfree related initialization */
+ HEAPfreeBuffer = malloc(HEAP_FREE_BUFFER_SIZE * sizeof(char*));
+ MT_lock_init(&HEAPfreeBufferLock, "heap_free_buffer_lock");
+ HEAPfreeBufferCount = 0;
+ MT_Id tid;
+ MT_create_thread(&tid, HEAPfreeWorker, NULL, MT_THR_DETACHED,
"HEAPfreeWorker");
+
ATOMIC_SET(&GDKstopped, 0);
mainpid = MT_getpid();
_______________________________________________
checkin-list mailing list -- [email protected]
To unsubscribe send an email to [email protected]
