I have another patchset here which refactors a little code and adds two
hooks for the ordering of devices for results to the create_open and
get_paths commands.
The 5th patch in this set shows the goal I'm moving towards, a plugin
that prioritizes local storage nodes (defined by the network zones
provided by the MogileFS::Network module(1))
Any comments on this design?
--hachi
(1) The MogileFS::Network module is originally from Andy McFarland on
this list October 3, 2007. Slightly modified now.
>From 04db156d1d1afb021f6c436843ae972cdfbd9d9c Mon Sep 17 00:00:00 2001
From: Jonathan Steinert <[EMAIL PROTECTED]>
Date: Tue, 11 Mar 2008 16:45:39 -0700
Subject: [PATCH] Switch to passing device objects around instead of device id's. This is a lead-in to a hook being able to handle the list.
---
server/lib/MogileFS/Worker/Query.pm | 14 +++++++-------
1 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/server/lib/MogileFS/Worker/Query.pm b/server/lib/MogileFS/Worker/Query.pm
index 1bb6884..f77445f 100644
--- a/server/lib/MogileFS/Worker/Query.pm
+++ b/server/lib/MogileFS/Worker/Query.pm
@@ -861,8 +861,6 @@ sub cmd_get_paths {
paths => 0,
};
- my @devices_with_weights;
-
# find devids that FID is on in memcache or db.
my @fid_devids;
my $need_devids_in_memcache = 0;
@@ -881,10 +879,13 @@ sub cmd_get_paths {
$memc->add($devid_memkey, [EMAIL PROTECTED], 3600) if $need_devids_in_memcache;
}
+ my @devices = map { $dmap->{$_} } @fid_devids;
+
+ my @devices_with_weights;
+
# is this fid still owned by this key?
- foreach my $devid (@fid_devids) {
+ foreach my $dev (@devices) {
my $weight;
- my $dev = $dmap->{$devid};
my $util = $dev->observed_utilization;
if (defined($util) and $util =~ /\A\d+\Z/) {
@@ -894,7 +895,7 @@ sub cmd_get_paths {
$weight = $dev->weight;
$weight ||= 100;
}
- push @devices_with_weights, [$devid, $weight];
+ push @devices_with_weights, [$dev, $weight];
}
# randomly weight the devices
@@ -904,8 +905,7 @@ sub cmd_get_paths {
my $backup_path;
# construct result paths
- foreach my $devid (@list) {
- my $dev = $dmap->{$devid};
+ foreach my $dev (@list) {
next unless $dev && ($dev->can_read_from);
my $host = $dev->host;
--
1.5.4.3
>From 0efe46cfd82f3ca78b8e7643e5bd2fe2092996b4 Mon Sep 17 00:00:00 2001
From: Jonathan Steinert <[EMAIL PROTECTED]>
Date: Tue, 11 Mar 2008 16:53:23 -0700
Subject: [PATCH] Move the sorting of devices by utilization into a seperate subroutine.
---
server/lib/MogileFS/Worker/Query.pm | 46 +++++++++++++++++++---------------
1 files changed, 26 insertions(+), 20 deletions(-)
diff --git a/server/lib/MogileFS/Worker/Query.pm b/server/lib/MogileFS/Worker/Query.pm
index f77445f..4f3da98 100644
--- a/server/lib/MogileFS/Worker/Query.pm
+++ b/server/lib/MogileFS/Worker/Query.pm
@@ -881,31 +881,13 @@ sub cmd_get_paths {
my @devices = map { $dmap->{$_} } @fid_devids;
- my @devices_with_weights;
-
- # is this fid still owned by this key?
- foreach my $dev (@devices) {
- my $weight;
- my $util = $dev->observed_utilization;
-
- if (defined($util) and $util =~ /\A\d+\Z/) {
- $weight = 102 - $util;
- $weight ||= 100;
- } else {
- $weight = $dev->weight;
- $weight ||= 100;
- }
- push @devices_with_weights, [$dev, $weight];
- }
-
- # randomly weight the devices
- my @list = MogileFS::Util::weighted_list(@devices_with_weights);
+ my @sorted_devs = sort_devs_by_utilization(@devices);
# keep one partially-bogus path around just in case we have nothing else to send.
my $backup_path;
# construct result paths
- foreach my $dev (@list) {
+ foreach my $dev (@sorted_devs) {
next unless $dev && ($dev->can_read_from);
my $host = $dev->host;
@@ -940,6 +922,30 @@ sub cmd_get_paths {
return $self->ok_line($ret);
}
+sub sort_devs_by_utilization {
+ my @devices_with_weights;
+
+ # is this fid still owned by this key?
+ foreach my $dev (@_) {
+ my $weight;
+ my $util = $dev->observed_utilization;
+
+ if (defined($util) and $util =~ /\A\d+\Z/) {
+ $weight = 102 - $util;
+ $weight ||= 100;
+ } else {
+ $weight = $dev->weight;
+ $weight ||= 100;
+ }
+ push @devices_with_weights, [$dev, $weight];
+ }
+
+ # randomly weight the devices
+ my @list = MogileFS::Util::weighted_list(@devices_with_weights);
+
+ return @list;
+}
+
# ------------------------------------------------------------
#
# NOTE: cmd_edit_file is EXPERIMENTAL. Please see the documentation
--
1.5.4.3
>From 80cacb5c0f12f381ffb05f18142fc5792fcc43d5 Mon Sep 17 00:00:00 2001
From: Jonathan Steinert <[EMAIL PROTECTED]>
Date: Tue, 11 Mar 2008 17:00:46 -0700
Subject: [PATCH] Add global hook for taking care of the ordering of devices at get_paths time.
---
server/lib/MogileFS/Worker/Query.pm | 5 ++++-
1 files changed, 4 insertions(+), 1 deletions(-)
diff --git a/server/lib/MogileFS/Worker/Query.pm b/server/lib/MogileFS/Worker/Query.pm
index 4f3da98..5348942 100644
--- a/server/lib/MogileFS/Worker/Query.pm
+++ b/server/lib/MogileFS/Worker/Query.pm
@@ -881,7 +881,10 @@ sub cmd_get_paths {
my @devices = map { $dmap->{$_} } @fid_devids;
- my @sorted_devs = sort_devs_by_utilization(@devices);
+ my @sorted_devs;
+ unless (MogileFS::run_global_hook('cmd_get_paths_order_devices', [EMAIL PROTECTED], [EMAIL PROTECTED])) {
+ @sorted_devs = sort_devs_by_utilization(@devices);
+ }
# keep one partially-bogus path around just in case we have nothing else to send.
my $backup_path;
--
1.5.4.3
>From d2b07d00276825cce166e20470aa858bfa24ba35 Mon Sep 17 00:00:00 2001
From: Jonathan Steinert <[EMAIL PROTECTED]>
Date: Tue, 1 Apr 2008 18:09:41 -0700
Subject: [PATCH] Refactor cmd_create_open to call into sort_devs_by_freespace and also call a hook to let a plugin override this sort order.
---
server/lib/MogileFS/Worker/Query.pm | 36 ++++++++++++++++++++++++++--------
1 files changed, 27 insertions(+), 9 deletions(-)
diff --git a/server/lib/MogileFS/Worker/Query.pm b/server/lib/MogileFS/Worker/Query.pm
index 5348942..d0eb9e8 100644
--- a/server/lib/MogileFS/Worker/Query.pm
+++ b/server/lib/MogileFS/Worker/Query.pm
@@ -241,21 +241,24 @@ sub cmd_create_open {
$profstart->("wait_monitor");
$self->wait_for_monitor;
+ $profstart->("find_deviceid");
+
+ my @devices;
+
+ unless (MogileFS::run_global_hook('cmd_create_open_order_devices', [MogileFS::Device->devices], [EMAIL PROTECTED])) {
+ @devices = sort_devs_by_freespace(MogileFS::Device->devices);
+ }
+
# find suitable device(s) to put this file on.
my @dests; # MogileFS::Device objects which are suitabke
- $profstart->("find_deviceid");
while (scalar(@dests) < ($multi ? 3 : 1)) {
- my $ddev = first {
- $_->should_get_new_files &&
- $_->not_on_hosts(map { $_->host } @dests)
- } weighted_list map {
- [$_, 100 * $_->percent_free]
- } grep {
- $_->exists
- } MogileFS::Device->devices;
+ my $ddev = shift @devices;
last unless $ddev;
+ next unless $ddev->should_get_new_files;
+ next unless $ddev->not_on_hosts(map { $_->host } @dests);
+
push @dests, $ddev;
}
return $self->err_line("no_devices") unless @dests;
@@ -319,6 +322,21 @@ sub cmd_create_open {
return $self->ok_line($res);
}
+sub sort_devs_by_freespace {
+ my @devices_with_weights;
+
+ foreach my $dev (@_) {
+ next unless $dev->exists;
+
+ my $weight = 100 * $dev->percent_free;
+ push @devices_with_weights, [$dev, $weight];
+ }
+
+ my @list = MogileFS::Util::weighted_list(@devices_with_weights);
+
+ return @list;
+}
+
sub cmd_create_close {
my MogileFS::Worker::Query $self = shift;
my $args = shift;
--
1.5.4.3
>From 4eab87c129139ef8ee340068241834208111acad Mon Sep 17 00:00:00 2001
From: Jonathan Steinert <[EMAIL PROTECTED]>
Date: Wed, 2 Apr 2008 23:52:50 -0700
Subject: [PATCH] Add ZoneLocal plugin which hooks in using new hooks to reorder devices for create_open and get_paths results.
---
.../lib/MogileFS/Plugin/ZoneLocal.pm | 66 ++++++++++++++++++++
1 files changed, 66 insertions(+), 0 deletions(-)
create mode 100644 server-plugins/MogileFS-Plugin-ZoneLocal/lib/MogileFS/Plugin/ZoneLocal.pm
diff --git a/server-plugins/MogileFS-Plugin-ZoneLocal/lib/MogileFS/Plugin/ZoneLocal.pm b/server-plugins/MogileFS-Plugin-ZoneLocal/lib/MogileFS/Plugin/ZoneLocal.pm
new file mode 100644
index 0000000..0cb5fe0
--- /dev/null
+++ b/server-plugins/MogileFS-Plugin-ZoneLocal/lib/MogileFS/Plugin/ZoneLocal.pm
@@ -0,0 +1,66 @@
+# FilePaths plugin for MogileFS, by hachi
+
+package MogileFS::Plugin::ZoneLocal;
+
+use strict;
+use warnings;
+
+our $VERSION = '0.01';
+$VERSION = eval $VERSION;
+
+use MogileFS::Worker::Query;
+use MogileFS::Network;
+
+sub prioritize_devs_current_zone;
+
+sub load {
+ MogileFS::register_global_hook( 'cmd_get_paths_order_devices', sub {
+ my $devices = shift;
+ my $sorted_devs = shift;
+
+ @$sorted_devs = prioritize_devs_current_zone
+ MogileFS::Worker::Query::sort_devs_by_utilization(@$devices);
+
+ return 1;
+ });
+
+ MogileFS::register_global_hook( 'cmd_create_open_order_devices', sub {
+ my $devices = shift;
+ my $sorted_devs = shift;
+
+ @$sorted_devs = prioritize_devs_current_zone
+ MogileFS::Worker::Query::sort_devs_by_freespace(@$devices);
+
+ return 1;
+ });
+
+ return 1;
+}
+
+sub unload {
+ # remove our hooks
+ MogileFS::unregister_global_hook( 'cmd_get_paths_order_devices' );
+
+ return 1;
+}
+
+sub prioritize_devs_current_zone {
+ my $current_zone = MogileFS::Network->zone_for_ip($MogileFS::REQ_client_ip);
+ my (@this_zone, @other_zone);
+
+ foreach my $dev (@_) {
+ my $ip = $dev->host->ip;
+ my $host_id = $dev->host->id;
+ my $zone = MogileFS::Network->zone_for_ip($ip);
+
+ if ($current_zone eq $zone) {
+ push @this_zone, $dev;
+ } else {
+ push @other_zone, $dev;
+ }
+ }
+
+ return @this_zone, @other_zone;
+}
+
+1;
--
1.5.4.3
