Hello community, here is the log from the commit of package perl-Minion for openSUSE:Factory checked in at 2016-02-25 22:04:34 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/perl-Minion (Old) and /work/SRC/openSUSE:Factory/.perl-Minion.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "perl-Minion" Changes: -------- --- /work/SRC/openSUSE:Factory/perl-Minion/perl-Minion.changes 2016-01-28 17:25:36.000000000 +0100 +++ /work/SRC/openSUSE:Factory/.perl-Minion.new/perl-Minion.changes 2016-02-26 00:38:57.000000000 +0100 @@ -1,0 +2,29 @@ +Fri Feb 19 10:32:17 UTC 2016 - [email protected] + +- updated to 5.0 + see /usr/share/doc/packages/perl-Minion/Changes + + 5.0 2016-02-17 + - Added start event to Minion::Job. + - Added -R option to worker command. + - Reduced default missing_after value to 30 minutes. + - Reduced default remove_after value to 2 days. + - Improved Minion::Backend::Pg performance significantly with a new index and + PostgreSQL 9.5 features. + - Improved Minion::Job to capture more exceptions. + - Improved worker command to support the QUIT signal. + - Improved worker command to repair in less regular intervals. + +------------------------------------------------------------------- +Sun Feb 7 10:15:51 UTC 2016 - [email protected] + +- updated to 4.06 + see /usr/share/doc/packages/perl-Minion/Changes + + 4.06 2016-02-06 + - Improved performance of Minion::Backend::Pg slightly. + + 4.05 2016-02-05 + - Improved Minion::Backend::Pg to check the PostgreSQL version. + +------------------------------------------------------------------- Old: ---- Minion-4.04.tar.gz New: ---- Minion-5.0.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ perl-Minion.spec ++++++ --- /var/tmp/diff_new_pack.3JTlkr/_old 2016-02-26 00:38:58.000000000 +0100 +++ /var/tmp/diff_new_pack.3JTlkr/_new 2016-02-26 00:38:58.000000000 +0100 @@ -17,7 +17,7 @@ Name: perl-Minion -Version: 4.04 +Version: 5.0 Release: 0 %define cpan_name Minion Summary: Job queue @@ -36,13 +36,14 @@ %description Minion is a job queue for the at http://mojolicious.org real-time web -framework with support for multiple named queues, priorities, delayed jobs, -job results, retries with backoff, distributed workers, parallel processing -and multiple backends (such as at http://www.postgresql.org). +framework, with support for multiple named queues, priorities, delayed +jobs, job results, retries with backoff, statistics, distributed workers, +parallel processing, autoscaling, resource leak protection and multiple +backends (such as at http://www.postgresql.org). -A job queue allows you to process time and/or computationally intensive -tasks in background processes, outside of the request/response lifecycle. -Among those tasks you'll commonly find image resizing, spam filtering, HTTP +Job queues allow you to process time and/or computationally intensive tasks +in background processes, outside of the request/response lifecycle. Among +those tasks you'll commonly find image resizing, spam filtering, HTTP downloads, building tarballs, warming caches and basically everything else you can imagine that's not super fast. ++++++ Minion-4.04.tar.gz -> Minion-5.0.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/Changes new/Minion-5.0/Changes --- old/Minion-4.04/Changes 2016-01-23 04:33:42.000000000 +0100 +++ new/Minion-5.0/Changes 2016-02-17 08:10:29.000000000 +0100 @@ -1,4 +1,21 @@ +5.0 2016-02-17 + - Added start event to Minion::Job. + - Added -R option to worker command. + - Reduced default missing_after value to 30 minutes. + - Reduced default remove_after value to 2 days. + - Improved Minion::Backend::Pg performance significantly with a new index and + PostgreSQL 9.5 features. + - Improved Minion::Job to capture more exceptions. + - Improved worker command to support the QUIT signal. + - Improved worker command to repair in less regular intervals. + +4.06 2016-02-06 + - Improved performance of Minion::Backend::Pg slightly. + +4.05 2016-02-05 + - Improved Minion::Backend::Pg to check the PostgreSQL version. + 4.04 2016-01-23 - Updated Minion::Backend::Pg to use new Mojo::Pg features. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/META.json new/Minion-5.0/META.json --- old/Minion-4.04/META.json 2016-01-23 04:38:50.000000000 +0100 +++ new/Minion-5.0/META.json 2016-02-17 14:41:59.000000000 +0100 @@ -1,5 +1,5 @@ { - "abstract" : "Job Queue", + "abstract" : "Job queue", "author" : [ "Sebastian Riedel <[email protected]>" ], @@ -54,5 +54,5 @@ }, "x_IRC" : "irc://irc.perl.org/#mojo" }, - "version" : "4.04" + "version" : "5.0" } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/META.yml new/Minion-5.0/META.yml --- old/Minion-4.04/META.yml 2016-01-23 04:38:50.000000000 +0100 +++ new/Minion-5.0/META.yml 2016-02-17 14:41:59.000000000 +0100 @@ -1,5 +1,5 @@ --- -abstract: 'Job Queue' +abstract: 'Job queue' author: - 'Sebastian Riedel <[email protected]>' build_requires: @@ -27,4 +27,4 @@ homepage: http://mojolicious.org license: http://www.opensource.org/licenses/artistic-license-2.0 repository: https://github.com/kraih/minion.git -version: '4.04' +version: '5.0' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/Makefile.PL new/Minion-5.0/Makefile.PL --- old/Minion-4.04/Makefile.PL 2016-01-03 01:02:28.000000000 +0100 +++ new/Minion-5.0/Makefile.PL 2016-02-16 22:27:01.000000000 +0100 @@ -8,7 +8,7 @@ WriteMakefile( NAME => 'Minion', VERSION_FROM => 'lib/Minion.pm', - ABSTRACT => 'Job Queue', + ABSTRACT => 'Job queue', AUTHOR => 'Sebastian Riedel <[email protected]>', LICENSE => 'artistic_2', META_MERGE => { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/README.md new/Minion-5.0/README.md --- old/Minion-4.04/README.md 2016-01-03 01:02:32.000000000 +0100 +++ new/Minion-5.0/README.md 2016-02-16 22:27:01.000000000 +0100 @@ -2,8 +2,16 @@ # Minion [](https://travis-ci.org/kraih/minion) A job queue for the [Mojolicious](http://mojolicious.org) real-time web - framework with support for multiple backends, such as - [PostgreSQL](http://www.postgresql.org). + framework, with support for multiple named queues, priorities, delayed jobs, + job results, retries with backoff, statistics, distributed workers, parallel + processing, autoscaling, resource leak protection and multiple backends (such + as [PostgreSQL](http://www.postgresql.org)). + + Job queues allow you to process time and/or computationally intensive tasks in + background processes, outside of the request/response lifecycle. Among those + tasks you'll commonly find image resizing, spam filtering, HTTP downloads, + building tarballs, warming caches and basically everything else you can + imagine that's not super fast. ```perl use Mojolicious::Lite; @@ -39,3 +47,7 @@ $ curl -L https://cpanmin.us | perl - -M https://cpan.metacpan.org -n Minion We recommend the use of a [Perlbrew](http://perlbrew.pl) environment. + +## Want to know more? + + Take a look at our excellent [documentation](http://mojolicious.org/perldoc>)! diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/lib/Minion/Backend/Pg.pm new/Minion-5.0/lib/Minion/Backend/Pg.pm --- old/Minion-4.04/lib/Minion/Backend/Pg.pm 2016-01-23 04:33:36.000000000 +0100 +++ new/Minion-5.0/lib/Minion/Backend/Pg.pm 2016-02-16 22:39:04.000000000 +0100 @@ -1,6 +1,7 @@ package Minion::Backend::Pg; use Mojo::Base 'Minion::Backend'; +use Carp 'croak'; use Mojo::IOLoop; use Mojo::Pg; use Sys::Hostname 'hostname'; @@ -24,9 +25,7 @@ } sub enqueue { - my ($self, $task) = (shift, shift); - my $args = shift // []; - my $options = shift // {}; + my ($self, $task, $args, $options) = (shift, shift, shift || [], shift || {}); my $db = $self->pg->db; return $db->query( @@ -59,8 +58,7 @@ 'select id from minion_jobs where (state = $1 or $1 is null) and (task = $2 or $2 is null) order by id desc - limit $3 - offset $4', @$options{qw(state task)}, $limit, $offset + limit $3 offset $4', @$options{qw(state task)}, $limit, $offset )->arrays->map(sub { $self->job_info($_->[0]) })->to_array; } @@ -74,20 +72,24 @@ sub new { my $self = shift->SUPER::new(pg => Mojo::Pg->new(@_)); + + croak 'PostgreSQL 9.5 or later is required' + if Mojo::Pg->new(@_)->db->dbh->{pg_server_version} < 90500; my $pg = $self->pg->auto_migrate(1)->max_connections(1); $pg->migrations->name('minion')->from_data; + return $self; } sub register_worker { my ($self, $id) = @_; - my $sql - = 'update minion_workers set notified = now() where id = ? returning 1'; - return $id if $id && $self->pg->db->query($sql, $id)->rows; - - $sql = 'insert into minion_workers (host, pid) values (?, ?) returning id'; - return $self->pg->db->query($sql, hostname, $$)->hash->{id}; + return shift->pg->db->query( + "insert into minion_workers (id, host, pid) + values (coalesce(?, nextval('minion_workers_id_seq')), ?, ?) + on conflict(id) do update set notified = now() + returning id", $id, $self->{host} //= hostname, $$ + )->hash->{id}; } sub remove_job { @@ -128,8 +130,7 @@ sub reset { shift->pg->db->query('truncate minion_jobs, minion_workers') } sub retry_job { - my ($self, $id, $retries) = (shift, shift, shift); - my $options = shift // {}; + my ($self, $id, $retries, $options) = (shift, shift, shift, shift || {}); return !!$self->pg->db->query( "update minion_jobs @@ -137,7 +138,7 @@ retried = now(), retries = retries + 1, state = 'inactive', delayed = (now() + (interval '1 second' * ?)) where id = ? and retries = ? - and state in ('failed', 'finished', 'inactive') + and state in ('inactive', 'failed', 'finished') returning 1", @$options{qw(priority queue)}, $options->{delay} // 0, $id, $retries )->rows; @@ -146,24 +147,18 @@ sub stats { my $self = shift; - my $db = $self->pg->db; - my $all = $db->query('select count(*) from minion_workers')->array->[0]; - my $sql - = "select count(distinct worker) from minion_jobs where state = 'active'"; - my $active = $db->query($sql)->array->[0]; - - $sql = 'select state, count(state) from minion_jobs group by 1'; - my $states - = $db->query($sql)->arrays->reduce(sub { $a->{$b->[0]} = $b->[1]; $a }, {}); - - return { - active_jobs => $states->{active} || 0, - active_workers => $active, - failed_jobs => $states->{failed} || 0, - finished_jobs => $states->{finished} || 0, - inactive_jobs => $states->{inactive} || 0, - inactive_workers => $all - $active - }; + my $stats = $self->pg->db->query( + "select state::text || '_jobs', count(state) from minion_jobs group by state + union all + select 'inactive_workers', count(*) from minion_workers + union all + select 'active_workers', count(distinct worker) from minion_jobs + where state = 'active'" + )->arrays->reduce(sub { $a->{$b->[0]} = $b->[1]; $a }, {}); + $stats->{inactive_workers} -= $stats->{active_workers}; + $stats->{"${_}_jobs"} ||= 0 for qw(inactive active failed finished); + + return $stats; } sub unregister_worker { @@ -193,7 +188,7 @@ and task = any (?) order by priority desc, created limit 1 - for update + for update skip locked ) returning id, args, retries, task", $id, $options->{queues} || ['default'], [keys %{$self->minion->tasks}] @@ -683,7 +678,6 @@ task text not null, worker bigint ); -create index on minion_jobs (priority desc, created); create table if not exists minion_workers ( id bigserial not null primary key, host text not null, @@ -715,14 +709,25 @@ notified timestamp with time zone not null default now(); alter table minion_workers alter column started set default now(); --- 3 up -create index on minion_jobs (state); - -- 4 up alter table minion_jobs add column queue text not null default 'default'; -- 5 up alter table minion_jobs add column attempts int not null default 1; --- 6 up -drop index minion_jobs_state_idx; +-- 7 up +create type minion_state as enum ('inactive', 'active', 'failed', 'finished'); +alter table minion_jobs alter column state set default 'inactive'::minion_state; +alter table minion_jobs + alter column state type minion_state using state::minion_state; +alter table minion_jobs alter column args type jsonb using args::jsonb; +alter table minion_jobs alter column result type jsonb using result::jsonb; + +-- 7 down +alter table minion_jobs alter column state type text using state; +alter table minion_jobs alter column state set default 'inactive'; +drop type if exists minion_state; + +-- 8 up +alter table minion_jobs add constraint args check(jsonb_typeof(args) = 'array'); +create index on minion_jobs (state, priority desc, created); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/lib/Minion/Command/minion/worker.pm new/Minion-5.0/lib/Minion/Command/minion/worker.pm --- old/Minion-4.04/lib/Minion/Command/minion/worker.pm 2016-01-17 17:47:30.000000000 +0100 +++ new/Minion-5.0/lib/Minion/Command/minion/worker.pm 2016-02-17 08:18:54.000000000 +0100 @@ -11,18 +11,21 @@ my ($self, @args) = @_; GetOptionsFromArray \@args, - 'I|heartbeat-interval=i' => \($self->{interval} = 60), - 'j|jobs=i' => \($self->{max} = 4), - 'q|queue=s' => \my @queues; + 'I|heartbeat-interval=i' => \($self->{hearthbeat} = 60), + 'j|jobs=i' => \($self->{max} = 4), + 'q|queue=s' => \my @queues, + 'R|repair-interval=i' => \($self->{repair} = 21600); $self->{queues} = @queues ? \@queues : ['default']; local $SIG{CHLD} = 'DEFAULT'; local $SIG{INT} = local $SIG{TERM} = sub { $self->{finished}++ }; + local $SIG{QUIT} + = sub { ++$self->{finished} and kill 'KILL', keys %{$self->{jobs}} }; # Log fatal errors my $app = $self->app; + $app->log->debug("Worker $$ started"); my $worker = $self->{worker} = $app->minion->worker; - @$self{qw(register repair)} = (0, 0); eval { $self->_work until $self->{finished} && !keys %{$self->{jobs}}; 1 } or $app->log->fatal("Worker error: $@"); $worker->unregister; @@ -33,16 +36,15 @@ # Send heartbeats in regular intervals my $worker = $self->{worker}; - $worker->register and $self->{register} = steady_time + $self->{interval} - if $self->{register} < steady_time; + $worker->register and $self->{register} = steady_time + $self->{hearthbeat} + if ($self->{register} || 0) < steady_time; - # Repair in regular intervals - if ($self->{repair} < steady_time) { + # Repair in regular intervals (randomize to avoid congestion) + if (($self->{check} || 0) < steady_time) { my $app = $self->app; $app->log->debug('Checking worker registry and job queue'); - my $minion = $app->minion; - $minion->repair; - $self->{repair} = steady_time + $minion->missing_after; + my $repair = $self->{repair}; + $self->{check} = steady_time + ($repair - int rand $repair / 2); } # Check if jobs are finished @@ -54,7 +56,10 @@ # Try to get more jobs elsif (my $job = $worker->dequeue(5 => {queues => $self->{queues}})) { - $jobs->{$job->start} = $job; + $jobs->{my $pid = $job->start} = $job; + my ($id, $task) = ($job->id, $job->task); + $self->app->log->debug( + qq{Performing job "$id" with task "$task" in process $pid}); } } @@ -71,7 +76,7 @@ Usage: APPLICATION minion worker [OPTIONS] ./myapp.pl minion worker - ./myapp.pl minion worker -m production -I 15 -j 10 + ./myapp.pl minion worker -m production -I 15 -R 3600 -j 10 ./myapp.pl minion worker -q important -q default Options: @@ -87,6 +92,7 @@ MOJO_MODE/PLACK_ENV or "development" -q, --queue <name> One or more queues to get jobs from, defaults to "default" + -R, --repair-interval <seconds> Repair interval, defaults to 21600 =head1 DESCRIPTION @@ -102,6 +108,10 @@ Stop gracefully after finishing the current jobs. +=head2 QUIT + +Stop immediately without finishing the current jobs. + =head1 ATTRIBUTES L<Minion::Command::minion::worker> inherits all attributes from diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/lib/Minion/Job.pm new/Minion-5.0/lib/Minion/Job.pm --- old/Minion-4.04/lib/Minion/Job.pm 2016-01-03 01:02:15.000000000 +0100 +++ new/Minion-5.0/lib/Minion/Job.pm 2016-02-16 22:27:01.000000000 +0100 @@ -9,9 +9,8 @@ sub app { shift->minion->app } sub fail { - my $self = shift; - my $err = shift // 'Unknown error'; - my $ok = $self->minion->backend->fail_job($self->id, $self->retries, $err); + my ($self, $err) = (shift, shift // 'Unknown error'); + my $ok = $self->minion->backend->fail_job($self->id, $self->retries, $err); return $ok ? !!$self->emit(failed => $err) : undef; } @@ -51,15 +50,15 @@ die "Can't fork: $!" unless defined(my $pid = fork); $self->emit(spawn => $pid) and return $pid if $pid; - # Reset event loop - Mojo::IOLoop->reset; - # Child - my $task = $self->task; - $self->app->log->debug( - qq{Performing job "@{[$self->id]}" with task "$task" in process $$}); - my $cb = $self->minion->tasks->{$task}; - $self->fail($@) unless eval { $self->$cb(@{$self->args}); 1 }; + eval { + + # Reset event loop + Mojo::IOLoop->reset; + $self->minion->tasks->{$self->emit('start')->task}->($self, @{$self->args}); + + 1; + } or $self->fail($@); exit 0; } @@ -133,6 +132,20 @@ say "Job $id running in process $pid"; }); +=head2 start + + $job->on(start => sub { + my $job = shift; + ... + }); + +Emitted in the process performing this job, after it has been spawned. + + $job->on(start => sub { + my $job = shift; + $0 = $job->id; + }); + =head1 ATTRIBUTES L<Minion::Job> implements the following attributes. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/lib/Minion.pm new/Minion-5.0/lib/Minion.pm --- old/Minion-4.04/lib/Minion.pm 2016-01-22 22:30:56.000000000 +0100 +++ new/Minion-5.0/lib/Minion.pm 2016-02-17 08:45:14.000000000 +0100 @@ -12,11 +12,11 @@ has app => sub { Mojo::Server->new->build_app('Mojo::HelloWorld') }; has 'backend'; has backoff => sub { \&_backoff }; -has missing_after => 86400; -has remove_after => 864000; +has missing_after => 1800; +has remove_after => 172800; has tasks => sub { {} }; -our $VERSION = '4.04'; +our $VERSION = '5.0'; sub add_task { ($_[0]->tasks->{$_[1]} = $_[2]) and return $_[0] } @@ -115,12 +115,13 @@ =head1 DESCRIPTION L<Minion> is a job queue for the L<Mojolicious|http://mojolicious.org> real-time -web framework with support for multiple named queues, priorities, delayed jobs, -job results, retries with backoff, distributed workers, parallel processing and -multiple backends (such as L<PostgreSQL|http://www.postgresql.org>). +web framework, with support for multiple named queues, priorities, delayed jobs, +job results, retries with backoff, statistics, distributed workers, parallel +processing, autoscaling, resource leak protection and multiple backends (such as +L<PostgreSQL|http://www.postgresql.org>). -A job queue allows you to process time and/or computationally intensive tasks -in background processes, outside of the request/response lifecycle. Among those +Job queues allow you to process time and/or computationally intensive tasks in +background processes, outside of the request/response lifecycle. Among those tasks you'll commonly find image resizing, spam filtering, HTTP downloads, building tarballs, warming caches and basically everything else you can imagine that's not super fast. @@ -248,7 +249,7 @@ Amount of time in seconds after which workers without a heartbeat will be considered missing and removed from the registry by L</"repair">, defaults to -C<86400> (1 day). +C<1800> (30 minutes). =head2 remove_after @@ -257,7 +258,7 @@ Amount of time in seconds after which jobs that have reached the state C<finished> will be removed automatically by L</"repair">, defaults to -C<864000> (10 days). +C<172800> (2 days). =head2 tasks diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-4.04/t/pg.t new/Minion-5.0/t/pg.t --- old/Minion-4.04/t/pg.t 2015-11-12 02:08:51.000000000 +0100 +++ new/Minion-5.0/t/pg.t 2016-02-16 22:27:01.000000000 +0100 @@ -23,6 +23,12 @@ my $worker = $minion->repair->worker; isa_ok $worker->minion->app, 'Mojolicious', 'has default application'; +# Migrate up and down +is $minion->backend->pg->migrations->active, 8, 'active version is 8'; +is $minion->backend->pg->migrations->migrate(0)->active, 0, + 'active version is 0'; +is $minion->backend->pg->migrations->migrate->active, 8, 'active version is 8'; + # Register and unregister $worker->register; like $worker->info->{started}, qr/^[\d.]+$/, 'has timestamp'; @@ -362,6 +368,13 @@ $job->on(failed => sub { $failed++ }); $job->on(finished => sub { $finished++ }); $job->on(spawn => sub { $pid = pop }); + $job->on( + start => sub { + my $job = shift; + return unless $job->task eq 'switcheroo'; + $job->task('add')->args->[-1] += 1; + } + ); } ); } @@ -388,6 +401,11 @@ is $err, "test\n", 'right error'; is $failed, 1, 'failed event has been emitted once'; is $finished, 1, 'finished event has been emitted once'; +$minion->add_task(switcheroo => sub { }); +$minion->enqueue(switcheroo => [5, 3]); +$job = $worker->dequeue(0); +$job->perform; +is_deeply $job->info->{result}, {added => 9}, 'right result'; $worker->unregister; # Queues
