Bartosz Zaborowski a écrit :
> Hi all,
>
>
Hi,
> I've rewritten some code of image loader to get real performance gain from
> threads on SMP machine. Threads support exist in geeqie since around rev
> 1100, but in fact, there are up to 2 simultaneously running threads - main
> (for GUI) and only 1 for image loader. And GUI doesn't do anything
> cpu-intensive.
>
> With this patch geeqie runs a few simultaneous threads for image loaders
> while it loads image for main view and makes thumbnails in main list.
> Number of threads depends on MAX_THREADS constant (currently located in
> main.h, but it should probably be determined in configure somehow - I'm not
> familiar with automake etc).
> Low prio tasks (making thumbnails) runs in up to MAX_THREADS threads. When
> high-prio task arrives, some of the low-prio are stopped - so when
> MAX_THREADS=1 you get exactly the old behavior.
>
> It seems that MAX_THREADS should be set to number_of_cpus+1, with that value
> I've got best performance.
>
> I haven't modified the search and duplicates dialogs with their thumbnails
> loading methods - maybe I'll rewrite them later (I don't understand why there
> is more than 1 separate mechanism for making thumbnails?).
>
> And I did some tests: after modification on 2core CPU I've got about 185% of
> initial performance, and when MAX_THREADS=1 we get sth which is not worse
> than original geeqie.
>
> There are also some minor modifications in file reading - I've added madvise
> call to force readahead of mmaped file (with many threads its really
> important) and increased a bit "read buffer" - it gives a little performance
> gain for free.
>
> I hope somebody will enjoy my work;)
>
> PS. I know its not the best time to post such patch, since you are releasing
> 1.0, but it's also a part of a project for my course at high school;)
> PS2. I'll be offline from 9.07 to ~28.07, so I won't answer for any comments
> those days.
>
> --
> bart
>
>
Thanks for this patch, i attached a version of it updated to match
latest revision (1796) for all to test.
I didn't have the time for now to test it in details, but it looks
interesting.
--
Zas
Index: src/view_file.c
===================================================================
--- src/view_file.c (révision 1796)
+++ src/view_file.c (copie de travail)
@@ -671,6 +671,7 @@
file_data_unref(vf->dir_fd);
g_free(vf->info);
+ g_mutex_free(vf->thumb_loader_mutex);
g_free(vf);
}
@@ -755,6 +756,8 @@
if (dir_fd) vf_set_fd(vf, dir_fd);
+ vf->thumb_loader_mutex = g_mutex_new();
+
return vf;
}
@@ -779,8 +782,11 @@
}
}
+/* for thumbnail loaders threads management */
+static gint thumb_loaders_threads = 0;
static gboolean vf_thumb_next(ViewFile *vf);
+static void vf_start_thumb_next_loaders(ViewFile *vf);
static gdouble vf_thumb_progress(ViewFile *vf)
{
@@ -814,26 +820,52 @@
}
}
-static void vf_thumb_do(ViewFile *vf, FileData *fd)
+static void vf_thumb_do(ViewFile *vf, FileData *fd, gint loader_i)
{
+ vf->thumbs_loader_busy[loader_i] = FALSE;
+ if (vf->thumbs_loader[loader_i])
+ {
+ thumb_loader_free(vf->thumbs_loader[loader_i]);
+ }
+
+ vf->thumbs_loader[loader_i] = NULL;
+ vf->thumbs_filedata[loader_i] = NULL;
+
if (!fd) return;
vf_set_thumb_fd(vf, fd);
+ fd->thumb_loading = FALSE;
vf_thumb_status(vf, vf_thumb_progress(vf), _("Loading thumbs..."));
}
void vf_thumb_cleanup(ViewFile *vf)
{
+ gint i;
vf_thumb_status(vf, 0.0, NULL);
vf->thumbs_running = FALSE;
- thumb_loader_free(vf->thumbs_loader);
- vf->thumbs_loader = NULL;
+ for (i=0; i < MAX_THREADS; i++)
+ {
+ if (vf->thumbs_loader_busy[i] && vf->thumbs_filedata[i])
+ {
+ vf->thumbs_filedata[i]->thumb_loading = FALSE;
+ }
+ thumb_loader_free(vf->thumbs_loader[i]);
+ vf->thumbs_loader[i] = NULL;
- vf->thumbs_filedata = NULL;
+ vf->thumbs_filedata[i] = NULL;
+ vf->thumbs_loader_busy[i] = FALSE;
+ }
+ thumb_loaders_threads = 0;
}
+void vf_thumb_cleanup_gently(ViewFile *vf)
+{
+ vf->thumbs_running = FALSE;
+ vf_thumb_status(vf, 0.0, NULL);
+}
+
void vf_thumb_stop(ViewFile *vf)
{
if (vf->thumbs_running) vf_thumb_cleanup(vf);
@@ -841,14 +873,20 @@
static void vf_thumb_common_cb(ThumbLoader *tl, gpointer data)
{
+ gint i;
ViewFile *vf = data;
- if (vf->thumbs_filedata && vf->thumbs_loader == tl)
+ for(i=0; i < MAX_THREADS; i++)
{
- vf_thumb_do(vf, vf->thumbs_filedata);
+ if (vf->thumbs_filedata[i] && vf->thumbs_loader[i] == tl)
+ {
+ vf_thumb_do(vf, vf->thumbs_filedata[i], i);
+ break;
+ }
}
+ thumb_loaders_threads--;
- while (vf_thumb_next(vf));
+ vf_start_thumb_next_loaders(vf);
}
static void vf_thumb_error_cb(ThumbLoader *tl, gpointer data)
@@ -861,46 +899,97 @@
vf_thumb_common_cb(tl, data);
}
+static void vf_start_thumb_next_loaders(ViewFile *vf)
+{
+ if (!GTK_WIDGET_REALIZED(vf->listview))
+ {
+ vf_thumb_status(vf, 0.0, NULL);
+ return ;
+ }
+
+ do
+ {
+ while (vf_thumb_next(vf) && vf->thumbs_running);
+ if (!vf->thumbs_running)
+ {
+ break;
+ }
+ }
+ while (thumb_loaders_threads < MAX_THREADS);
+}
+
static gboolean vf_thumb_next(ViewFile *vf)
{
FileData *fd = NULL;
+ gint loader_i;
- if (!GTK_WIDGET_REALIZED(vf->listview))
+/*
+ * critical section begin
+ * FIXME: do calbacks run in the callers thread? If always in main thread - this mutex can be removed
+ */
+ g_mutex_lock(vf->thumb_loader_mutex);
+
+ /* find first index of not busy thumbs_loader and mark it busy */
+ for (loader_i=0; loader_i < MAX_THREADS && vf->thumbs_loader_busy[loader_i]; loader_i++);
+ if (loader_i == MAX_THREADS)
{
- vf_thumb_status(vf, 0.0, NULL);
+ /* this should not happen */
+ g_mutex_unlock(vf->thumb_loader_mutex);
+ DEBUG_1("======================================\nsth is going REALLY WRONG!\n");
return FALSE;
}
+ vf->thumbs_loader_busy[loader_i] = TRUE;
+ thumb_loaders_threads++;
+
+ /* since thumb_next can be run from concurrent threads (??), this must be under mutex too */
switch (vf->type)
{
case FILEVIEW_LIST: fd = vflist_thumb_next_fd(vf); break;
case FILEVIEW_ICON: fd = vficon_thumb_next_fd(vf); break;
}
+ /* mark this fd as already taken */
+ if (fd)
+ fd->thumb_loading = TRUE;
+
+ g_mutex_unlock(vf->thumb_loader_mutex);
+/* critical section end */
+
if (!fd)
{
/* done */
- vf_thumb_cleanup(vf);
+ thumb_loaders_threads--;
+ if (thumb_loaders_threads > 0)
+ {
+ vf_thumb_cleanup_gently(vf); /* only stop the 'thumbnails loop' */
+ }
+ else
+ {
+ vf_thumb_cleanup(vf); /* cleanup everything */
+ }
+
return FALSE;
}
- vf->thumbs_filedata = fd;
- thumb_loader_free(vf->thumbs_loader);
- vf->thumbs_loader = thumb_loader_new(options->thumbnails.max_width, options->thumbnails.max_height);
- thumb_loader_set_callbacks(vf->thumbs_loader,
+ vf->thumbs_filedata[loader_i] = fd;
+
+ vf->thumbs_loader[loader_i] = thumb_loader_new(options->thumbnails.max_width, options->thumbnails.max_height);
+ thumb_loader_set_callbacks(vf->thumbs_loader[loader_i],
vf_thumb_done_cb,
vf_thumb_error_cb,
NULL,
vf);
- if (!thumb_loader_start(vf->thumbs_loader, fd))
+ if (!thumb_loader_start(vf->thumbs_loader[loader_i], fd))
{
/* set icon to unknown, continue */
DEBUG_1("thumb loader start failed %s", fd->path);
- vf_thumb_do(vf, fd);
+ vf_thumb_do(vf, fd, loader_i);
+ thumb_loaders_threads--;
return TRUE;
}
@@ -931,7 +1020,7 @@
thumb_format_changed = FALSE;
}
- while (vf_thumb_next(vf));
+ vf_start_thumb_next_loaders(vf);
}
Index: src/image-load.c
===================================================================
--- src/image-load.c (révision 1796)
+++ src/image-load.c (copie de travail)
@@ -22,8 +22,8 @@
#include <fcntl.h>
#include <sys/mman.h>
-#define IMAGE_LOADER_READ_BUFFER_SIZE_DEFAULT 4096
-#define IMAGE_LOADER_IDLE_READ_LOOP_COUNT_DEFAULT 1
+#define IMAGE_LOADER_READ_BUFFER_SIZE_DEFAULT 65536
+#define IMAGE_LOADER_IDLE_READ_LOOP_COUNT_DEFAULT 1
/**************************************************************************************/
@@ -651,8 +651,8 @@
close(load_fd);
return FALSE;
}
-
- il->mapped_file = mmap(0, il->bytes_total, PROT_READ|PROT_WRITE, MAP_PRIVATE, load_fd, 0);
+
+ il->mapped_file = mmap(0, il->bytes_total, PROT_READ, MAP_PRIVATE, load_fd, 0);
close(load_fd);
if (il->mapped_file == MAP_FAILED)
{
@@ -661,7 +661,9 @@
}
il->preview = FALSE;
}
-
+
+ madvise(il->mapped_file, il->bytes_total, MADV_WILLNEED);
+
return TRUE;
}
@@ -786,6 +788,7 @@
static GCond *image_loader_prio_cond = NULL;
static GMutex *image_loader_prio_mutex = NULL;
static gint image_loader_prio_num = 0;
+static gint image_loader_low_prio_num = 0;
static void image_loader_thread_enter_high(void)
@@ -799,22 +802,40 @@
{
g_mutex_lock(image_loader_prio_mutex);
image_loader_prio_num--;
- if (image_loader_prio_num == 0) g_cond_broadcast(image_loader_prio_cond); /* wake up all low prio threads */
+ if (image_loader_prio_num < MAX_THREADS) g_cond_broadcast(image_loader_prio_cond); /* wake up all low prio threads */
g_mutex_unlock(image_loader_prio_mutex);
}
static void image_loader_thread_wait_high(void)
{
g_mutex_lock(image_loader_prio_mutex);
- while (image_loader_prio_num)
+ image_loader_low_prio_num--;
+ while (image_loader_prio_num + image_loader_low_prio_num >= MAX_THREADS)
{
g_cond_wait(image_loader_prio_cond, image_loader_prio_mutex);
}
+ image_loader_low_prio_num++;
+ g_mutex_unlock(image_loader_prio_mutex);
+}
+static void image_loader_thread_enter_low(void)
+{
+ g_mutex_lock(image_loader_prio_mutex);
+ image_loader_low_prio_num++;
g_mutex_unlock(image_loader_prio_mutex);
+ image_loader_thread_wait_high();
}
+static void image_loader_thread_leave_low(void)
+{
+ g_mutex_lock(image_loader_prio_mutex);
+ image_loader_low_prio_num--;
+ g_cond_broadcast(image_loader_prio_cond); /* wake up all low prio threads */
+ g_mutex_unlock(image_loader_prio_mutex);
+}
+extern gint thumb_loaders_threads;
+
static void image_loader_thread_run(gpointer data, gpointer user_data)
{
ImageLoader *il = data;
@@ -822,15 +843,15 @@
if (il->idle_priority > G_PRIORITY_DEFAULT_IDLE)
{
- /* low prio, wait untill high prio tasks finishes */
- image_loader_thread_wait_high();
+ /* low prio, wait until high prio tasks finishes */
+ image_loader_thread_enter_low();
}
else
{
/* high prio */
image_loader_thread_enter_high();
}
-
+
cont = image_loader_begin(il);
if (!cont && !image_loader_get_pixbuf(il))
@@ -854,11 +875,15 @@
}
image_loader_stop_loader(il);
+ /* wakeup next thread */
if (il->idle_priority <= G_PRIORITY_DEFAULT_IDLE)
{
- /* high prio */
image_loader_thread_leave_high();
}
+ else
+ {
+ image_loader_thread_leave_low();
+ }
g_mutex_lock(il->data_mutex);
il->can_destroy = TRUE;
Index: src/main.h
===================================================================
--- src/main.h (révision 1796)
+++ src/main.h (copie de travail)
@@ -109,12 +109,16 @@
"%formatted.Aperture%|%formatted.ShutterSpeed%|%formatted.ISOSpeedRating:ISO *%|%formatted.FocalLength%|%formatted.ExposureBias:* Ev%\n" \
"%formatted.Camera:40%|%formatted.Flash%"
+#define MAX_THREADS 3
+
#define GQ_LINK_STR "â"
#include "typedefs.h"
#include "debug.h"
#include "options.h"
#define DESKTOP_FILE_TEMPLATE GQ_APP_DIR "/template.desktop"
+
+
/*
*----------------------------------------------------------------------------
* main.c
Index: src/view_file_list.c
===================================================================
--- src/view_file_list.c (révision 1796)
+++ src/view_file_list.c (copie de travail)
@@ -1096,7 +1096,7 @@
gtk_tree_model_get(store, &iter, FILE_COLUMN_POINTER, &nfd, -1);
- if (!nfd->thumb_pixbuf) fd = nfd;
+ if (!nfd->thumb_pixbuf && !nfd->thumb_loading) fd = nfd;
valid = gtk_tree_model_iter_next(store, &iter);
}
@@ -1119,7 +1119,7 @@
while (work2 && !fd)
{
fd_p = work2->data;
- if (!fd_p->thumb_pixbuf) fd = fd_p;
+ if (!fd_p->thumb_pixbuf && !fd_p->thumb_loading) fd = fd_p;
work2 = work2->next;
}
}
@@ -1141,6 +1141,7 @@
{
g_object_unref(fd->thumb_pixbuf);
fd->thumb_pixbuf = NULL;
+ fd->thumb_loading = FALSE;
}
work = work->next;
}
Index: src/view_file_icon.c
===================================================================
--- src/view_file_icon.c (révision 1796)
+++ src/view_file_icon.c (copie de travail)
@@ -1842,7 +1842,7 @@
while (!fd && list)
{
IconData *id = list->data;
- if (id && !id->fd->thumb_pixbuf) fd = id->fd;
+ if (id && !id->fd->thumb_pixbuf && !id->fd->thumb_loading) fd = id->fd;
list = list->next;
}
@@ -1861,7 +1861,7 @@
FileData *fd_p = id->fd;
work = work->next;
- if (!fd_p->thumb_pixbuf) fd = fd_p;
+ if (!fd_p->thumb_pixbuf && !fd_p->thumb_loading) fd = fd_p;
}
}
@@ -1880,6 +1880,7 @@
{
g_object_unref(fd->thumb_pixbuf);
fd->thumb_pixbuf = NULL;
+ fd->thumb_loading = FALSE;
}
work = work->next;
}
Index: src/typedefs.h
===================================================================
--- src/typedefs.h (révision 1796)
+++ src/typedefs.h (copie de travail)
@@ -490,6 +490,9 @@
ExifData *exif;
GHashTable *modified_xmp; // hash table which contains unwritten xmp metadata in format: key->list of string values
+
+ gboolean thumb_loading; /* for multithreaded thumbnails loading, set to TRUE means, that regardless of
+ thumb_pixbuf == NULL this file should not be selected for creating next thumb */
};
struct _LayoutOptions
@@ -743,8 +746,10 @@
/* thumbs updates*/
gboolean thumbs_running;
- ThumbLoader *thumbs_loader;
- FileData *thumbs_filedata;
+ ThumbLoader *thumbs_loader[MAX_THREADS];
+ FileData *thumbs_filedata[MAX_THREADS];
+ gboolean thumbs_loader_busy[MAX_THREADS];
+ GMutex *thumb_loader_mutex;
/* marks */
gboolean marks_enabled;
------------------------------------------------------------------------------
Enter the BlackBerry Developer Challenge
This is your chance to win up to $100,000 in prizes! For a limited time,
vendors submitting new applications to BlackBerry App World(TM) will have
the opportunity to enter the BlackBerry Developer Challenge. See full prize
details at: http://p.sf.net/sfu/blackberry
_______________________________________________
Geeqie-devel mailing list
Geeqie-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/geeqie-devel
