[Cluster-devel] [PATCH][GFS2] Bouncing locks in a cluster is slow in GFS2
Hi,
This patch is a performance improvement for GFS2 in a clustered
environment. It makes the glock hold time self-adjusting.
Regards,
Bob Peterson
Red Hat File Systems
Signed-off-by: Bob Peterson rpete...@redhat.com
Bouncing locks in a cluster is slow in GFS2
--
fs/gfs2/glock.c | 89 --
fs/gfs2/glock.h |6
fs/gfs2/glops.c |2 -
fs/gfs2/incore.h |2 +-
4 files changed, 73 insertions(+), 26 deletions(-)
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index c75d499..117d8e2 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -58,7 +58,6 @@ static int __dump_glock(struct seq_file *seq, const struct
gfs2_glock *gl);
static void do_xmote(struct gfs2_glock *gl, struct gfs2_holder *gh, unsigned
int target);
static struct dentry *gfs2_root;
-static struct workqueue_struct *glock_workqueue;
struct workqueue_struct *gfs2_delete_workqueue;
static LIST_HEAD(lru_list);
static atomic_t lru_count = ATOMIC_INIT(0);
@@ -67,9 +66,23 @@ static DEFINE_SPINLOCK(lru_lock);
#define GFS2_GL_HASH_SHIFT 15
#define GFS2_GL_HASH_SIZE (1 GFS2_GL_HASH_SHIFT)
#define GFS2_GL_HASH_MASK (GFS2_GL_HASH_SIZE - 1)
+#define GL_WORKQUEUES0x2
+#define GL_WQ_MASK 0x1
static struct hlist_bl_head gl_hash_table[GFS2_GL_HASH_SIZE];
static struct dentry *gfs2_root;
+static struct workqueue_struct *glock_workqueue[GL_WORKQUEUES];
+
+static inline int qwork(struct gfs2_glock *gl, unsigned long delay)
+{
+ struct workqueue_struct *wq;
+
+ wq = glock_workqueue[gl-gl_name.ln_type GL_WQ_MASK];
+
+ if (gl-gl_name.ln_type != LM_TYPE_INODE)
+ delay = 0;
+ return queue_delayed_work(wq, gl-gl_work, delay);
+}
/**
* gl_hash() - Turn glock number into hash bucket number
@@ -407,6 +420,10 @@ static void state_change(struct gfs2_glock *gl, unsigned
int new_state)
if (held1 held2 list_empty(gl-gl_holders))
clear_bit(GLF_QUEUED, gl-gl_flags);
+ if (new_state != gl-gl_target)
+ /* shorten our minimum hold time */
+ gl-gl_hold_time = max(gl-gl_hold_time - GL_GLOCK_HOLD_DECR,
+ GL_GLOCK_MIN_HOLD);
gl-gl_state = new_state;
gl-gl_tchange = jiffies;
}
@@ -550,7 +567,7 @@ __acquires(gl-gl_spin)
GLOCK_BUG_ON(gl, ret);
} else { /* lock_nolock */
finish_xmote(gl, target);
- if (queue_delayed_work(glock_workqueue, gl-gl_work, 0) == 0)
+ if (qwork(gl, 0) == 0)
gfs2_glock_put(gl);
}
@@ -623,7 +640,7 @@ out_sched:
clear_bit(GLF_LOCK, gl-gl_flags);
smp_mb__after_clear_bit();
gfs2_glock_hold(gl);
- if (queue_delayed_work(glock_workqueue, gl-gl_work, 0) == 0)
+ if (qwork(gl, 0) == 0)
gfs2_glock_put_nolock(gl);
return;
@@ -670,15 +687,14 @@ static void glock_work_func(struct work_struct *work)
gl-gl_state != LM_ST_UNLOCKED
gl-gl_demote_state != LM_ST_EXCLUSIVE) {
unsigned long holdtime, now = jiffies;
- holdtime = gl-gl_tchange + gl-gl_ops-go_min_hold_time;
+ holdtime = gl-gl_tchange + gl-gl_hold_time;
if (time_before(now, holdtime))
delay = holdtime - now;
set_bit(delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE, gl-gl_flags);
}
run_queue(gl, 0);
spin_unlock(gl-gl_spin);
- if (!delay ||
- queue_delayed_work(glock_workqueue, gl-gl_work, delay) == 0)
+ if (!delay || qwork(gl, delay) == 0)
gfs2_glock_put(gl);
if (drop_ref)
gfs2_glock_put(gl);
@@ -741,6 +757,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
gl-gl_tchange = jiffies;
gl-gl_object = NULL;
gl-gl_sbd = sdp;
+ gl-gl_hold_time = GL_GLOCK_DFT_HOLD;
INIT_DELAYED_WORK(gl-gl_work, glock_work_func);
INIT_WORK(gl-gl_delete, delete_work_func);
@@ -852,8 +869,15 @@ static int gfs2_glock_demote_wait(void *word)
static void wait_on_holder(struct gfs2_holder *gh)
{
+ unsigned long time1 = jiffies;
+
might_sleep();
wait_on_bit(gh-gh_iflags, HIF_WAIT, gfs2_glock_holder_wait,
TASK_UNINTERRUPTIBLE);
+ if (time_after(jiffies, time1 + HZ)) /* have we waited a second? */
+ /* Lengthen the minimum hold time. */
+ gh-gh_gl-gl_hold_time = min(gh-gh_gl-gl_hold_time +
+ GL_GLOCK_HOLD_INCR,
+ GL_GLOCK_MAX_HOLD);
}
static void wait_on_demote(struct gfs2_glock *gl)
@@ -1087,8 +,8 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
gfs2_glock_hold(gl);
if (test_bit(GLF_PENDING_DEMOTE, gl-gl_flags)
!test_bit(GLF_DEMOTE, gl-gl_flags))
- delay =
Re: [Cluster-devel] [PATCH][GFS2] Bouncing locks in a cluster is slow in GFS2 - Try #2
- Original Message -
| Hi,
|
| You shouldn't need to do the dual workqueue trick upstream since the
| workqueues will already start as many threads as required (so even
| better than just using the two here). If that isn't happening we
| should
| ask Tejun about it.
|
| So I think we only need the min hold time bit here,
|
| Steve.
Hi,
Based on your feedback, I reworked this patch a bit. This is take two.
Rather than using a qwork function to police the delay based on
glock type, I went back to the original queue_delayed_work and
adjusted delay based on glock type. So this version has fewer lines
changed, but still accomplishes the same thing.
Note that I had to tweak function glock_work_func in order to
preserve the rather odd logic that decides whether or not to
actually queue the work.
This patch is a performance improvement for GFS2 in a clustered
environment. It makes the glock hold time self-adjusting.
Regards,
Bob Peterson
Red Hat File Systems
Signed-off-by: Bob Peterson rpete...@redhat.com
Bouncing locks in a cluster is slow in GFS2
--
[Cluster-devel] [PATCH][GFS2] Bouncing locks in a cluster is slow in GFS2
fs/gfs2/glock.c | 39 +--
fs/gfs2/glock.h |6 ++
fs/gfs2/glops.c |2 --
fs/gfs2/incore.h |2 +-
4 files changed, 36 insertions(+), 13 deletions(-)
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index c75d499..0523b20 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -407,6 +407,10 @@ static void state_change(struct gfs2_glock *gl, unsigned
int new_state)
if (held1 held2 list_empty(gl-gl_holders))
clear_bit(GLF_QUEUED, gl-gl_flags);
+ if (new_state != gl-gl_target)
+ /* shorten our minimum hold time */
+ gl-gl_hold_time = max(gl-gl_hold_time - GL_GLOCK_HOLD_DECR,
+ GL_GLOCK_MIN_HOLD);
gl-gl_state = new_state;
gl-gl_tchange = jiffies;
}
@@ -670,16 +674,21 @@ static void glock_work_func(struct work_struct *work)
gl-gl_state != LM_ST_UNLOCKED
gl-gl_demote_state != LM_ST_EXCLUSIVE) {
unsigned long holdtime, now = jiffies;
- holdtime = gl-gl_tchange + gl-gl_ops-go_min_hold_time;
+ holdtime = gl-gl_tchange + gl-gl_hold_time;
if (time_before(now, holdtime))
delay = holdtime - now;
set_bit(delay ? GLF_PENDING_DEMOTE : GLF_DEMOTE, gl-gl_flags);
}
run_queue(gl, 0);
spin_unlock(gl-gl_spin);
- if (!delay ||
- queue_delayed_work(glock_workqueue, gl-gl_work, delay) == 0)
+ if (!delay)
gfs2_glock_put(gl);
+ else {
+ if (gl-gl_name.ln_type != LM_TYPE_INODE)
+ delay = 0;
+ if (queue_delayed_work(glock_workqueue, gl-gl_work, delay) ==
0)
+ gfs2_glock_put(gl);
+ }
if (drop_ref)
gfs2_glock_put(gl);
}
@@ -741,6 +750,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
gl-gl_tchange = jiffies;
gl-gl_object = NULL;
gl-gl_sbd = sdp;
+ gl-gl_hold_time = GL_GLOCK_DFT_HOLD;
INIT_DELAYED_WORK(gl-gl_work, glock_work_func);
INIT_WORK(gl-gl_delete, delete_work_func);
@@ -852,8 +862,15 @@ static int gfs2_glock_demote_wait(void *word)
static void wait_on_holder(struct gfs2_holder *gh)
{
+ unsigned long time1 = jiffies;
+
might_sleep();
wait_on_bit(gh-gh_iflags, HIF_WAIT, gfs2_glock_holder_wait,
TASK_UNINTERRUPTIBLE);
+ if (time_after(jiffies, time1 + HZ)) /* have we waited a second? */
+ /* Lengthen the minimum hold time. */
+ gh-gh_gl-gl_hold_time = min(gh-gh_gl-gl_hold_time +
+ GL_GLOCK_HOLD_INCR,
+ GL_GLOCK_MAX_HOLD);
}
static void wait_on_demote(struct gfs2_glock *gl)
@@ -1086,8 +1103,9 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
gfs2_glock_hold(gl);
if (test_bit(GLF_PENDING_DEMOTE, gl-gl_flags)
- !test_bit(GLF_DEMOTE, gl-gl_flags))
- delay = gl-gl_ops-go_min_hold_time;
+ !test_bit(GLF_DEMOTE, gl-gl_flags)
+ gl-gl_name.ln_type == LM_TYPE_INODE)
+ delay = gl-gl_hold_time;
if (queue_delayed_work(glock_workqueue, gl-gl_work, delay) == 0)
gfs2_glock_put(gl);
}
@@ -1270,12 +1288,13 @@ void gfs2_glock_cb(struct gfs2_glock *gl, unsigned int
state)
unsigned long now = jiffies;
gfs2_glock_hold(gl);
- holdtime = gl-gl_tchange + gl-gl_ops-go_min_hold_time;
- if (test_bit(GLF_QUEUED, gl-gl_flags)) {
+ holdtime = gl-gl_tchange + gl-gl_hold_time;
+ if (test_bit(GLF_QUEUED, gl-gl_flags)
+ gl-gl_name.ln_type == LM_TYPE_INODE) {
if (time_before(now, holdtime))
