We'll give workqueues a broadcast mechanism to ensure all
workers see a certain message. We'll also tag each worker
with {-wq_worker_nr} in preparation for work distribution.
This is intended to avoid extra connection and fork() costs
from LeiAuth in a future commit.
---
lib/PublicInbox/IPC.pm | 30 ++++++++++++++++++++++++----
lib/PublicInbox/WQWorker.pm | 8 ++++----
t/ipc.t | 39 ++++++++++++++++++++++---------------
3 files changed, 53 insertions(+), 24 deletions(-)
diff --git a/lib/PublicInbox/IPC.pm b/lib/PublicInbox/IPC.pm
index efac4c4d..2aeb6462 100644
--- a/lib/PublicInbox/IPC.pm
+++ b/lib/PublicInbox/IPC.pm
@@ -240,8 +240,9 @@ sub recv_and_run {
}
sub wq_worker_loop ($) {
- my ($self) = @_;
+ my ($self, $bcast_a) = @_;
my $wqw = PublicInbox::WQWorker->new($self);
+ PublicInbox::WQWorker->new($self, '-wq_bcast2');
PublicInbox::DS->SetPostLoopCallback(sub { $wqw->{sock} });
PublicInbox::DS->EventLoop;
PublicInbox::DS->Reset;
@@ -252,6 +253,20 @@ sub do_sock_stream { # via wq_io_do, for big requests
recv_and_run($self, delete $self->{0}, $len, 1);
}
+sub wq_broadcast {
+ my ($self, $sub, @args) = @_;
+ if (my $wkr = $self->{-wq_workers}) {
+ for my $bcast1 (values %$wkr) {
+ my $buf = ipc_freeze([$sub, @args]);
+ send($bcast1, $buf, MSG_EOR) // croak "send: $!";
+ # XXX shouldn't have to deal with EMSGSIZE here...
+ }
+ } else {
+ eval { $self->$sub(@args) };
+ warn "wq_broadcast: $@" if $@;
+ }
+}
+
sub wq_io_do { # always async
my ($self, $sub, $ios, @args) = @_;
if (my $s1 = $self->{-wq_s1}) { # run in worker
@@ -284,15 +299,21 @@ sub wq_io_do { # always async
sub _wq_worker_start ($$$) {
my ($self, $oldset, $fields) = @_;
+ my ($bcast1, $bcast2);
+ socketpair($bcast1, $bcast2, AF_UNIX, $SEQPACKET, 0) or
+ die "socketpair: $!";
my $seed = rand(0xffffffff);
my $pid = fork // die "fork: $!";
if ($pid == 0) {
srand($seed);
+ undef $bcast1;
eval { PublicInbox::DS->Reset };
- delete @$self{qw(-wq_s1 -wq_workers -wq_ppid)};
+ delete @$self{qw(-wq_s1 -wq_ppid)};
+ $self->{-wq_worker_nr} =
+ keys %{delete($self->{-wq_workers}) // {}};
$SIG{$_} = 'IGNORE' for (qw(PIPE));
$SIG{$_} = 'DEFAULT' for (qw(TTOU TTIN TERM QUIT INT CHLD));
- local $0 = $self->{-wq_ident};
+ local $0 = "$self->{-wq_ident} $self->{-wq_worker_nr}";
# ensure we properly exit even if warn() dies:
my $end = PublicInbox::OnDestroy->new($$, sub { exit(!!$@) });
eval {
@@ -301,12 +322,13 @@ sub _wq_worker_start ($$$) {
my $on_destroy = $self->ipc_atfork_child;
local %SIG = %SIG;
PublicInbox::DS::sig_setmask($oldset);
+ $self->{-wq_bcast2} = $bcast2;
wq_worker_loop($self);
};
warn "worker $self->{-wq_ident} PID:$$ died: $@" if $@;
undef $end; # trigger exit
} else {
- $self->{-wq_workers}->{$pid} = \undef;
+ $self->{-wq_workers}->{$pid} = $bcast1;
}
}
diff --git a/lib/PublicInbox/WQWorker.pm b/lib/PublicInbox/WQWorker.pm
index 25a5e4fb..3636321e 100644
--- a/lib/PublicInbox/WQWorker.pm
+++ b/lib/PublicInbox/WQWorker.pm
@@ -11,10 +11,10 @@ use Errno qw(EAGAIN ECONNRESET);
use IO::Handle (); # blocking
sub new {
- my (undef, $wq) = @_;
- my $s2 = $wq->{-wq_s2} // die 'BUG: no -wq_s2';
+ my ($cls, $wq, $field) = @_;
+ my $s2 = $wq->{$field // '-wq_s2'} // die "BUG: no {$field}";
$s2->blocking(0);
- my $self = bless { sock => $s2, wq => $wq }, __PACKAGE__;
+ my $self = bless { sock => $s2, wq => $wq }, $cls;
$self->SUPER::new($s2, EPOLLEXCLUSIVE|EPOLLIN|EPOLLET);
$self;
}
@@ -27,7 +27,7 @@ sub event_step {
} while ($n);
return if !defined($n) && $! == EAGAIN; # likely
warn "wq worker error: $!\n" if !defined($n) && $! != ECONNRESET;
- $self->{wq}->wq_atexit_child;
+ $self->{wq}->wq_atexit_child if $self->{sock} == $self->{wq}->{-wq_s2};
$self->close; # PublicInbox::DS::close
}
diff --git a/t/ipc.t b/t/ipc.t
index 345024bd..ca88eb59 100644
--- a/t/ipc.t
+++ b/t/ipc.t
@@ -9,6 +9,7 @@ use Fcntl qw(SEEK_SET);
use Digest::SHA qw(sha1_hex);
require_mods(qw(Storable||Sereal));
require_ok 'PublicInbox::IPC';
+my ($tmpdir, $for_destroy) = tmpdir();
state $once = eval <<'';
package PublicInbox::IPC;
use strict;
@@ -31,6 +32,12 @@ sub test_sha {
print { $self->{1} } sha1_hex($buf), "\n";
$self->{1}->flush;
}
+sub test_append_pid {
+ my ($self, $file) = @_;
+ open my $fh, '>>', $file or die "open: $!";
+ $fh->autoflush(1);
+ print $fh "$$\n" or die "print: $!";
+}
1;
my $ipc = bless {}, 'PublicInbox::IPC';
@@ -83,11 +90,8 @@ $test->('local');
defined($pid) or BAIL_OUT 'no spawn, no test';
is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
$test->('worker');
- {
- my ($tmp, $for_destroy) = tmpdir();
- $ipc->ipc_lock_init("$tmp/lock");
- is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
- }
+ $ipc->ipc_lock_init("$tmpdir/lock");
+ is($ipc->ipc_do('test_pid'), $pid, 'worker pid returned');
$ipc->ipc_worker_stop;
ok(!kill(0, $pid) && $!{ESRCH}, 'worker stopped');
}
@@ -167,18 +171,21 @@ SKIP: {
$SIG{__WARN__} = 'DEFAULT';
is($ipc->wq_workers_start('wq', 1), $$, 'workers started again');
is($ipc->wq_workers, 1, '1 worker started');
- SKIP: {
- $ipc->WQ_MAX_WORKERS > 1 or
- skip 'Inline::C or Socket::MsgHdr not available', 4;
- $ipc->wq_worker_incr;
- is($ipc->wq_workers, 2, 'worker count bumped');
- $ipc->wq_worker_decr;
- $ipc->wq_worker_decr_wait(10);
- is($ipc->wq_workers, 1, 'worker count lowered');
- is($ipc->wq_workers(2), 2, 'worker count set');
- is($ipc->wq_workers, 2, 'worker count stayed set');
- }
+
+ $ipc->wq_worker_incr;
+ is($ipc->wq_workers, 2, 'worker count bumped');
+ $ipc->wq_worker_decr;
+ $ipc->wq_worker_decr_wait(10);
+ is($ipc->wq_workers, 1, 'worker count lowered');
+ is($ipc->wq_workers(2), 2, 'worker count set');
+ is($ipc->wq_workers, 2, 'worker count stayed set');
+
+ $ipc->wq_broadcast('test_append_pid', "$tmpdir/append_pid");
$ipc->wq_close;
+ open my $fh, '<', "$tmpdir/append_pid" or BAIL_OUT "open: $!";
+ chomp(my @pids = <$fh>);
+ my %pids = map { $_ => 1 } grep(/\A[0-9]+\z/, @pids);
+ is(scalar keys %pids, 2, 'broadcast hit both PIDs');
is($ipc->wq_workers, undef, 'workers undef after close');
}
--
unsubscribe: one-click, see List-Unsubscribe header
archive: https://public-inbox.org/meta/
