Hello community, here is the log from the commit of package perl-Minion for openSUSE:Factory checked in at 2016-05-25 21:29:35 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/perl-Minion (Old) and /work/SRC/openSUSE:Factory/.perl-Minion.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "perl-Minion" Changes: -------- --- /work/SRC/openSUSE:Factory/perl-Minion/perl-Minion.changes 2016-04-12 19:39:02.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.perl-Minion.new/perl-Minion.changes 2016-05-25 21:29:36.000000000 +0200 @@ -1,0 +2,41 @@ +Mon May 23 12:11:00 UTC 2016 - [email protected] + +- updated to 5.08 + see /usr/share/doc/packages/perl-Minion/Changes + + 5.08 2016-05-20 + - Improved repair methods not to remove finished jobs with unresolved + dependencies. + + 5.07 2016-05-17 + - Added support for job dependencies. (jberger, sri) + - Added parents option to enqueue methods in Minion and Minion::Backend::Pg. + (jberger, sri) + - Added children and parents fields to info method in Minion::Job and job_info + method in Minion::Backend::Pg. + - Added -P option to job command. + - Improved stats methods to include jobs with unresolved dependencies in + delayed_jobs count. + + 5.06 2016-05-05 + - Improved worker command to support the TTIN, TTOU and USR1 signals. + - Improved Minion::Backend::Pg to handle delayed and retried jobs more + efficiently. + +------------------------------------------------------------------- +Thu May 5 09:38:08 UTC 2016 - [email protected] + +- updated to 5.05 + see /usr/share/doc/packages/perl-Minion/Changes + + 5.05 2016-04-20 + - Added queue option to list_jobs method in Minion::Backend::Pg. + - Improved performance of stats method in Minion::Backend::Pg slightly. + + 5.04 2016-04-19 + - Added EXPERIMENTAL delayed_jobs field to stats method in Minion and + Minion::Backend::Pg. + - Updated Mojo::Pg requirement to 2.18. + - Improved job command to show more detailed information for jobs and workers. + +------------------------------------------------------------------- Old: ---- Minion-5.03.tar.gz New: ---- Minion-5.08.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ perl-Minion.spec ++++++ --- /var/tmp/diff_new_pack.Sm2UUS/_old 2016-05-25 21:29:37.000000000 +0200 +++ /var/tmp/diff_new_pack.Sm2UUS/_new 2016-05-25 21:29:37.000000000 +0200 @@ -17,7 +17,7 @@ Name: perl-Minion -Version: 5.03 +Version: 5.08 Release: 0 %define cpan_name Minion Summary: Job queue @@ -37,9 +37,9 @@ %description Minion is a job queue for the at http://mojolicious.org real-time web framework, with support for multiple named queues, priorities, delayed -jobs, job results, retries with backoff, statistics, distributed workers, -parallel processing, autoscaling, resource leak protection and multiple -backends (such as at http://www.postgresql.org). +jobs, job dependencies, job results, retries with backoff, statistics, +distributed workers, parallel processing, autoscaling, resource leak +protection and multiple backends (such as at http://www.postgresql.org). Job queues allow you to process time and/or computationally intensive tasks in background processes, outside of the request/response lifecycle. Among @@ -101,6 +101,6 @@ %files -f %{name}.files %defattr(-,root,root,755) -%doc Changes CONTRIBUTING.md LICENSE README.md +%doc Changes examples LICENSE README.md %changelog ++++++ Minion-5.03.tar.gz -> Minion-5.08.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/CONTRIBUTING.md new/Minion-5.08/CONTRIBUTING.md --- old/Minion-5.03/CONTRIBUTING.md 2016-01-03 01:01:29.000000000 +0100 +++ new/Minion-5.08/CONTRIBUTING.md 1970-01-01 01:00:00.000000000 +0100 @@ -1 +0,0 @@ -Please read the guide for [contributing to Mojolicious](http://mojolicious.org/perldoc/Mojolicious/Guides/Contributing), Minion is a spin-off project and follows the same rules. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/Changes new/Minion-5.08/Changes --- old/Minion-5.03/Changes 2016-04-10 18:57:18.000000000 +0200 +++ new/Minion-5.08/Changes 2016-05-19 23:33:33.000000000 +0200 @@ -1,4 +1,33 @@ +5.08 2016-05-20 + - Improved repair methods not to remove finished jobs with unresolved + dependencies. + +5.07 2016-05-17 + - Added support for job dependencies. (jberger, sri) + - Added parents option to enqueue methods in Minion and Minion::Backend::Pg. + (jberger, sri) + - Added children and parents fields to info method in Minion::Job and job_info + method in Minion::Backend::Pg. + - Added -P option to job command. + - Improved stats methods to include jobs with unresolved dependencies in + delayed_jobs count. + +5.06 2016-05-05 + - Improved worker command to support the TTIN, TTOU and USR1 signals. + - Improved Minion::Backend::Pg to handle delayed and retried jobs more + efficiently. + +5.05 2016-04-20 + - Added queue option to list_jobs method in Minion::Backend::Pg. + - Improved performance of stats method in Minion::Backend::Pg slightly. + +5.04 2016-04-19 + - Added EXPERIMENTAL delayed_jobs field to stats methods in Minion and + Minion::Backend::Pg. + - Updated Mojo::Pg requirement to 2.18. + - Improved job command to show more detailed information for jobs and workers. + 5.03 2016-04-10 - Added enqueue event to Minion. (jberger) @@ -9,6 +38,7 @@ - Fixed worker command to repair in regular intervals. 5.0 2016-02-17 + - Minion::Backend::Pg now requires PostgreSQL 9.5. - Added start event to Minion::Job. - Added -R option to worker command. - Reduced default missing_after value to 30 minutes. @@ -57,7 +87,7 @@ - Added support for retrying failed jobs automatically. - Added backoff attribute to Minion. - Added attempts attribute to Minion::Job. - - Added attempts option to enqueue method in Minion and Minion::Backend::Pg. + - Added attempts option to enqueue methods in Minion and Minion::Backend::Pg. - Added -A option to job command. 3.0 2015-10-30 @@ -89,9 +119,8 @@ - Added queue option to enqueue method in Minion. - Added queue option to enqueue and retry_job methods in Minion::Backend::File and Minion::Backend::Pg. - - Added queues option to dequeue method in Minion::Worker. - - Added queues option to dequeue method in Minion::Backend::File and - Minion::Backend::Pg. + - Added queues option to dequeue methods in Minion::Worker, + Minion::Backend::File and Minion::Backend::Pg. - Added -q option to job and worker commands. - Improved worker command to be more resilient to time jumps. - Fixed a race condition in Minion::Backend::File and Minion::Backend::Pg @@ -101,7 +130,7 @@ 1.19 2015-09-28 - Added support for retrying jobs with a new priority. - Added priority option to retry method in Minion::Job. - - Added priority option to retry_job method in Minion::Backend::File and + - Added priority option to retry_job methods in Minion::Backend::File and Minion::Backend::Pg. 1.18 2015-08-30 @@ -118,7 +147,7 @@ 1.15 2015-05-15 - Added support for retrying jobs with a delay. (kwa) - Added delay option to retry method in Minion::Job. (kwa) - - Added delay option to retry_job method in Minion::Backend::File and + - Added delay option to retry_job methods in Minion::Backend::File and Minion::Backend::Pg. (kwa) 1.14 2015-04-21 @@ -159,7 +188,7 @@ - Improved commands to use less punctuation. 1.05 2015-01-05 - - Improved repair method in Minion::Backend::File and Minion::Backend::Pg to + - Improved repair methods in Minion::Backend::File and Minion::Backend::Pg to mention the current process in results of failed jobs. 1.04 2015-01-03 @@ -239,7 +268,7 @@ Minion::Backend::Mango and Minion::Worker. 0.29 2014-07-07 - - Renamed restart_job methods to retry_job in Minion::Backend, + - Renamed restart_job method to retry_job in Minion::Backend, Minion::Backend::File and Minion::Backend::Mango. - Renamed restart method to retry in Minion::Job. - Improved worker command to repair in regular intervals. @@ -284,8 +313,7 @@ ids. 0.13 2014-06-03 - - Added list_workers method to Minion::Backend. - - Added list_workers method to Minion::Backend::Mango. + - Added list_workers methods to Minion::Backend and Minion::Backend::Mango. 0.12 2014-06-03 - Fixed enqueue to use the correct time format. @@ -301,8 +329,7 @@ - Added modules Minion::Backend and Minion::Backend::Mango. - Added backend attribute to Minion. - Added reset method to Minion. - - Added info method to Minion::Job. - - Added info method to Minion::Worker. + - Added info methods to Minion::Job and Minion::Worker. - Added -L and -S options to job command. 0.09 2014-04-05 @@ -348,7 +375,7 @@ - Added support for delayed jobs. - Added stats method to Minion. - Added app method to Minion::Job. - - Reduced Perl version requirement to 5.10.1. + - Reduced Perl requirement to 5.10.1. 0.01 2014-03-27 - First release. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/MANIFEST new/Minion-5.08/MANIFEST --- old/Minion-5.03/MANIFEST 2016-04-10 19:02:44.000000000 +0200 +++ new/Minion-5.08/MANIFEST 2016-05-20 02:33:37.000000000 +0200 @@ -1,5 +1,5 @@ Changes -CONTRIBUTING.md +examples/minion_bench.pl lib/Minion.pm lib/Minion/Backend.pm lib/Minion/Backend/Pg.pm diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/META.json new/Minion-5.08/META.json --- old/Minion-5.03/META.json 2016-04-10 19:02:44.000000000 +0200 +++ new/Minion-5.08/META.json 2016-05-20 02:33:37.000000000 +0200 @@ -4,7 +4,7 @@ "Sebastian Riedel <[email protected]>" ], "dynamic_config" : 0, - "generated_by" : "ExtUtils::MakeMaker version 7.0401, CPAN::Meta::Converter version 2.150001", + "generated_by" : "ExtUtils::MakeMaker version 7.1001, CPAN::Meta::Converter version 2.150005", "license" : [ "artistic_2" ], @@ -54,5 +54,6 @@ }, "x_IRC" : "irc://irc.perl.org/#mojo" }, - "version" : "5.03" + "version" : "5.08", + "x_serialization_backend" : "JSON::PP version 2.27300" } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/META.yml new/Minion-5.08/META.yml --- old/Minion-5.03/META.yml 2016-04-10 19:02:44.000000000 +0200 +++ new/Minion-5.08/META.yml 2016-05-20 02:33:37.000000000 +0200 @@ -7,7 +7,7 @@ configure_requires: ExtUtils::MakeMaker: '0' dynamic_config: 0 -generated_by: 'ExtUtils::MakeMaker version 7.0401, CPAN::Meta::Converter version 2.150001' +generated_by: 'ExtUtils::MakeMaker version 7.1001, CPAN::Meta::Converter version 2.150005' license: artistic_2 meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html @@ -27,4 +27,5 @@ homepage: http://mojolicious.org license: http://www.opensource.org/licenses/artistic-license-2.0 repository: https://github.com/kraih/minion.git -version: '5.03' +version: '5.08' +x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/examples/minion_bench.pl new/Minion-5.08/examples/minion_bench.pl --- old/Minion-5.03/examples/minion_bench.pl 1970-01-01 01:00:00.000000000 +0100 +++ new/Minion-5.08/examples/minion_bench.pl 2016-05-17 03:16:18.000000000 +0200 @@ -0,0 +1,71 @@ +use Mojo::Base -strict; + +use Minion; +use Time::HiRes 'time'; + +my $ENQUEUE = 10000; +my $DEQUEUE = 1000; +my $REPETITIONS = 2; +my $WORKERS = 4; +my $STATS = 100; +my $REPAIR = 100; + +# A benchmark script for comparing backends and evaluating the performance +# impact of proposed changes +my $minion = Minion->new(Pg => 'postgresql://tester:testing@/test'); +$minion->add_task(foo => sub { }); +$minion->add_task(bar => sub { }); +$minion->reset; + +# Enqueue +say "Clean start with $ENQUEUE jobs"; +my $before = time; +$minion->enqueue($_ % 2 ? 'foo' : 'bar') for 1 .. $ENQUEUE; +my $elapsed = time - $before; +my $avg = sprintf '%.3f', $ENQUEUE / $elapsed; +say "Enqueued $ENQUEUE jobs in $elapsed seconds ($avg/s)"; + +# Dequeue +sub dequeue { + my @pids; + for (1 .. $WORKERS) { + die "Couldn't fork: $!" unless defined(my $pid = fork); + unless ($pid) { + my $worker = $minion->worker->register; + say "$$ will finish $DEQUEUE jobs"; + my $before = time; + $worker->dequeue(0.5)->finish for 1 .. $DEQUEUE; + my $elapsed = time - $before; + my $avg = sprintf '%.3f', $DEQUEUE / $elapsed; + say "$$ finished $DEQUEUE jobs in $elapsed seconds ($avg/s)"; + $worker->unregister; + exit; + } + push @pids, $pid; + } + + say "$$ has started $WORKERS workers"; + my $before = time; + waitpid $_, 0 for @pids; + my $elapsed = time - $before; + my $avg = sprintf '%.3f', ($DEQUEUE * $WORKERS) / $elapsed; + say + "$WORKERS workers finished $DEQUEUE jobs each in $elapsed seconds ($avg/s)"; +} +dequeue() for 1 .. $REPETITIONS; + +# Stats +say "Requesting stats $STATS times"; +$before = time; +$minion->stats for 1 .. $STATS; +$elapsed = time - $before; +$avg = sprintf '%.3f', $STATS / $elapsed; +say "Received stats $STATS times in $elapsed seconds ($avg/s)"; + +# Repair +say "Repairing $REPAIR times"; +$before = time; +$minion->repair for 1 .. $REPAIR; +$elapsed = time - $before; +$avg = sprintf '%.3f', $REPAIR / $elapsed; +say "Repaired $REPAIR times in $elapsed seconds ($avg/s)"; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Backend/Pg.pm new/Minion-5.08/lib/Minion/Backend/Pg.pm --- old/Minion-5.03/lib/Minion/Backend/Pg.pm 2016-02-21 23:36:42.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Backend/Pg.pm 2016-05-19 17:21:43.000000000 +0200 @@ -3,7 +3,7 @@ use Carp 'croak'; use Mojo::IOLoop; -use Mojo::Pg; +use Mojo::Pg 2.18; use Sys::Hostname 'hostname'; has 'pg'; @@ -29,11 +29,12 @@ my $db = $self->pg->db; return $db->query( - "insert into minion_jobs (args, attempts, delayed, priority, queue, task) - values (?, ?, (now() + (interval '1 second' * ?)), ?, ?, ?) + "insert into minion_jobs + (args, attempts, delayed, parents, priority, queue, task) + values (?, ?, (now() + (interval '1 second' * ?)), ?, ?, ?, ?) returning id", {json => $args}, $options->{attempts} // 1, - $options->{delay} // 0, $options->{priority} // 0, - $options->{queue} // 'default', $task + $options->{delay} // 0, $options->{parents} || [], + $options->{priority} // 0, $options->{queue} // 'default', $task )->hash->{id}; } @@ -42,12 +43,14 @@ sub job_info { shift->pg->db->query( - 'select id, args, attempts, extract(epoch from created) as created, + 'select id, args, attempts, + array(select id from minion_jobs where j.id = any(parents)) as children, + extract(epoch from created) as created, extract(epoch from delayed) as delayed, - extract(epoch from finished) as finished, priority, queue, result, - extract(epoch from retried) as retried, retries, + extract(epoch from finished) as finished, parents, priority, queue, + result, extract(epoch from retried) as retried, retries, extract(epoch from started) as started, state, task, worker - from minion_jobs where id = ?', shift + from minion_jobs as j where id = ?', shift )->expand->hash; } @@ -56,9 +59,10 @@ return $self->pg->db->query( 'select id from minion_jobs - where (state = $1 or $1 is null) and (task = $2 or $2 is null) + where (queue = $1 or $1 is null) and (state = $2 or $2 is null) + and (task = $3 or $3 is null) order by id desc - limit $3 offset $4', @$options{qw(state task)}, $limit, $offset + limit $4 offset $5', @$options{qw(queue state task)}, $limit, $offset )->arrays->map(sub { $self->job_info($_->[0]) })->to_array; } @@ -103,7 +107,7 @@ sub repair { my $self = shift; - # Check worker registry + # Workers without heartbeat my $db = $self->pg->db; my $minion = $self->minion; $db->query( @@ -111,19 +115,31 @@ where notified < now() - interval '1 second' * ?", $minion->missing_after ); - # Abandoned jobs + # Jobs with missing worker (can be retried) my $fail = $db->query( "select id, retries from minion_jobs as j where state = 'active' - and not exists(select 1 from minion_workers where id = j.worker)" + and not exists (select 1 from minion_workers where id = j.worker)" )->hashes; $fail->each(sub { $self->fail_job(@$_{qw(id retries)}, 'Worker went away') }); - # Old jobs + # Jobs with missing parents (can't be retried) $db->query( - "delete from minion_jobs - where state = 'finished' and finished < now() - interval '1 second' * ?", - $minion->remove_after + "update minion_jobs as j + set finished = now(), result = to_json('Parent went away'::text), + state = 'failed' + where parents <> '{}' and cardinality(parents) <> ( + select count(*) from minion_jobs where id = any(j.parents) + ) and state = 'inactive'" + ); + + # Old jobs with no unresolved dependencies + $db->query( + "delete from minion_jobs as j + where finished < now() - interval '1 second' * ? and not exists ( + select 1 from minion_jobs + where j.id = any(parents) and state <> 'finished' + ) and state = 'finished'", $minion->remove_after ); } @@ -134,12 +150,12 @@ return !!$self->pg->db->query( "update minion_jobs - set priority = coalesce(?, priority), queue = coalesce(?, queue), - retried = now(), retries = retries + 1, state = 'inactive', - delayed = (now() + (interval '1 second' * ?)) + set delayed = (now() + (interval '1 second' * ?)), + priority = coalesce(?, priority), queue = coalesce(?, queue), + retried = now(), retries = retries + 1, state = 'inactive' where id = ? and retries = ? and state in ('inactive', 'failed', 'finished') - returning 1", @$options{qw(priority queue)}, $options->{delay} // 0, $id, + returning 1", $options->{delay} // 0, @$options{qw(priority queue)}, $id, $retries )->rows; } @@ -148,7 +164,10 @@ my $self = shift; my $stats = $self->pg->db->query( - "select state::text || '_jobs', count(state) from minion_jobs group by state + "select state::text || '_jobs', count(*) from minion_jobs group by state + union all + select 'delayed_jobs', count(*) from minion_jobs + where (delayed > now() or parents <> '{}') and state = 'inactive' union all select 'inactive_workers', count(*) from minion_workers union all @@ -183,9 +202,11 @@ "update minion_jobs set started = now(), state = 'active', worker = ? where id = ( - select id from minion_jobs - where delayed <= now() and queue = any (?) and state = 'inactive' - and task = any (?) + select id from minion_jobs as j + where delayed <= now() and (parents = '{}' or cardinality(parents) = ( + select count(*) from minion_jobs + where id = any(j.parents) and state = 'finished' + )) and queue = any (?) and state = 'inactive' and task = any (?) order by priority desc, created limit 1 for update skip locked @@ -316,7 +337,8 @@ attempts => 25 -Number of times performing this job will be attempted, defaults to C<1>. +Number of times performing this job will be attempted, with a delay based on +L<Minion/"backoff"> after the first attempt, defaults to C<1>. =item delay @@ -324,6 +346,13 @@ Delay job for this many seconds (from now). +=item parents + + parents => [$id1, $id2, $id3] + +One or more existing jobs this job depends on, and that need to have +transitioned to the state C<finished> before it can be processed. + =item priority priority => 5 @@ -384,25 +413,37 @@ attempts => 25 -Number of times performing this job will be attempted, defaults to C<1>. +Number of times performing this job will be attempted. + +=item children + + children => ['10026', '10027', '10028'] + +Jobs depending on this job. =item created created => 784111777 -Time job was created. +Epoch time job was created. =item delayed delayed => 784111777 -Time job was delayed to. +Epoch time job was delayed to. =item finished finished => 784111777 -Time job was finished. +Epoch time job was finished. + +=item parents + + parents => ['10023', '10024', '10025'] + +Jobs this job depends on. =item priority @@ -426,7 +467,7 @@ retried => 784111777 -Time job has been retried. +Epoch time job has been retried. =item retries @@ -438,7 +479,7 @@ started => 784111777 -Time job was started. +Epoch time job was started. =item state @@ -471,6 +512,12 @@ =over 2 +=item queue + + queue => 'important' + +List only jobs in this queue. + =item state state => 'inactive' @@ -576,6 +623,14 @@ Number of workers that are currently processing a job. +=item delayed_jobs + + delayed_jobs => 100 + +Number of jobs in C<inactive> state that are scheduled to run at specific time +in the future or have unresolved dependencies. Note that this field is +EXPERIMENTAL and might change without warning! + =item failed_jobs failed_jobs => 100 @@ -637,7 +692,7 @@ notified => 784111777 -Last time worker sent a heartbeat. +Epoch time worker sent the last heartbeat. =item pid @@ -649,7 +704,7 @@ started => 784111777 -Time worker was started. +Epoch time worker was started. =back @@ -684,21 +739,9 @@ pid int not null, started timestamp with time zone not null ); -create or replace function minion_jobs_insert_notify() returns trigger as $$ - begin - perform pg_notify('minion.job', ''); - return null; - end; -$$ language plpgsql; -set client_min_messages to warning; -drop trigger if exists minion_jobs_insert_trigger on minion_jobs; -set client_min_messages to notice; -create trigger minion_jobs_insert_trigger after insert on minion_jobs - for each row execute procedure minion_jobs_insert_notify(); -- 1 down drop table if exists minion_jobs; -drop function if exists minion_jobs_insert_notify(); drop table if exists minion_workers; -- 2 up @@ -731,3 +774,27 @@ -- 8 up alter table minion_jobs add constraint args check(jsonb_typeof(args) = 'array'); create index on minion_jobs (state, priority desc, created); + +-- 9 up +create or replace function minion_jobs_notify_workers() returns trigger as $$ + begin + if new.delayed <= now() then + notify "minion.job"; + end if; + return null; + end; +$$ language plpgsql; +set client_min_messages to warning; +drop trigger if exists minion_jobs_insert_trigger on minion_jobs; +drop trigger if exists minion_jobs_notify_workers_trigger on minion_jobs; +set client_min_messages to notice; +create trigger minion_jobs_notify_workers_trigger + after insert or update of retries on minion_jobs + for each row execute procedure minion_jobs_notify_workers(); + +-- 9 down +drop trigger if exists minion_jobs_notify_workers_trigger on minion_jobs; +drop function if exists minion_jobs_notify_workers(); + +-- 10 up +alter table minion_jobs add column parents bigint[] default '{}'::bigint[]; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Backend.pm new/Minion-5.08/lib/Minion/Backend.pm --- old/Minion-5.03/lib/Minion/Backend.pm 2016-01-03 01:01:54.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Backend.pm 2016-05-17 20:59:35.000000000 +0200 @@ -146,7 +146,8 @@ attempts => 25 -Number of times performing this job will be attempted, defaults to C<1>. +Number of times performing this job will be attempted, with a delay based on +L<Minion/"backoff"> after the first attempt, defaults to C<1>. =item delay @@ -154,6 +155,13 @@ Delay job for this many seconds (from now). +=item parents + + parents => [$id1, $id2, $id3] + +One or more existing jobs this job depends on, and that need to have +transitioned to the state C<finished> before it can be processed. + =item priority priority => 5 @@ -216,25 +224,37 @@ attempts => 25 -Number of times performing this job will be attempted, defaults to C<1>. +Number of times performing this job will be attempted. + +=item children + + children => ['10026', '10027', '10028'] + +Jobs depending on this job. =item created created => 784111777 -Time job was created. +Epoch time job was created. =item delayed delayed => 784111777 -Time job was delayed to. +Epoch time job was delayed to. =item finished finished => 784111777 -Time job was finished. +Epoch time job was finished. + +=item parents + + parents => ['10023', '10024', '10025'] + +Jobs this job depends on. =item priority @@ -258,7 +278,7 @@ retried => 784111777 -Time job has been retried. +Epoch time job has been retried. =item retries @@ -270,7 +290,7 @@ started => 784111777 -Time job was started. +Epoch time job was started. =item state @@ -304,6 +324,12 @@ =over 2 +=item queue + + queue => 'important' + +List only jobs in this queue. + =item state state => 'inactive' @@ -408,6 +434,14 @@ Number of workers that are currently processing a job. +=item delayed_jobs + + delayed_jobs => 100 + +Number of jobs in C<inactive> state that are scheduled to run at specific time +in the future or have unresolved dependencies. Note that this field is +EXPERIMENTAL and might change without warning! + =item failed_jobs failed_jobs => 100 @@ -470,7 +504,7 @@ notified => 784111777 -Last time worker sent a heartbeat. +Epoch time worker sent the last heartbeat. =item pid @@ -482,7 +516,7 @@ started => 784111777 -Time worker was started. +Epoch time worker was started. =back diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Command/minion/job.pm new/Minion-5.08/lib/Minion/Command/minion/job.pm --- old/Minion-5.03/lib/Minion/Command/minion/job.pm 2016-01-13 19:11:03.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Command/minion/job.pm 2016-05-17 07:05:02.000000000 +0200 @@ -2,7 +2,6 @@ use Mojo::Base 'Mojolicious::Command'; use Getopt::Long qw(GetOptionsFromArray :config no_auto_abbrev no_ignore_case); -use Mojo::Date; use Mojo::JSON 'decode_json'; use Mojo::Util qw(dumper tablify); @@ -18,8 +17,9 @@ 'a|args=s' => sub { $args = decode_json($_[1]) }, 'd|delay=i' => \$options->{delay}, 'e|enqueue=s' => \my $enqueue, - 'l|limit=i' => \(my $limit = 100), - 'o|offset=i' => \(my $offset = 0), + 'l|limit=i' => \(my $limit = 100), + 'o|offset=i' => \(my $offset = 0), + 'P|parent=s' => ($options->{parents} = []), 'p|priority=i' => \$options->{priority}, 'q|queue=s' => \$options->{queue}, 'R|retry' => \my $retry, @@ -35,7 +35,8 @@ # Show stats or list jobs/workers return $self->_stats if $stats; - return $self->_list_workers($offset, $limit) if $workers; + return $id ? $self->_worker($id) : $self->_list_workers($offset, $limit) + if $workers; return $self->_list_jobs($offset, $limit, $options) unless defined $id; die "Job does not exist.\n" unless my $job = $self->app->minion->job($id); @@ -46,28 +47,7 @@ return $job->retry($options) || die "Job is active.\n" if $retry; # Job info - $self->_info($job); -} - -sub _date { Mojo::Date->new(@_)->to_datetime } - -sub _info { - my ($self, $job) = @_; - - # Details - my $info = $job->info; - my ($queue, $state, $priority, $retries) - = @$info{qw(queue state priority retries)}; - say $info->{task}, " ($queue, $state, p$priority, r$retries)"; - print dumper $info->{args}; - if (my $result = $info->{result}) { print dumper $result } - - # Timing - my ($created, $delayed) = @$info{qw(created delayed)}; - say _date($created), ' (created)'; - say _date($delayed), ' (delayed)' if $delayed > $created; - $info->{$_} and say _date($info->{$_}), " ($_)" - for qw(retried started finished); + print dumper $job->info; } sub _list_jobs { @@ -77,24 +57,16 @@ sub _list_workers { my $workers = shift->app->minion->backend->list_workers(@_); - print tablify [map { [_worker($_)] } @$workers]; + my @workers = map { [$_->{id}, $_->{host} . ':' . $_->{pid}] } @$workers; + print tablify \@workers; } -sub _stats { - my $stats = shift->app->minion->stats; - say "Inactive workers: $stats->{inactive_workers}"; - say "Active workers: $stats->{active_workers}"; - say "Inactive jobs: $stats->{inactive_jobs}"; - say "Active jobs: $stats->{active_jobs}"; - say "Failed jobs: $stats->{failed_jobs}"; - say "Finished jobs: $stats->{finished_jobs}"; -} +sub _stats { print dumper shift->app->minion->stats } sub _worker { - my $worker = shift; - my $state = @{$worker->{jobs}} ? 'active' : 'inactive'; - my $name = $worker->{host} . ':' . $worker->{pid}; - return $worker->{id}, $state, $name, _date($worker->{notified}); + die "Worker does not exist.\n" + unless my $worker = shift->app->minion->backend->worker_info(@_); + print dumper $worker; } 1; @@ -110,17 +82,18 @@ Usage: APPLICATION minion job [OPTIONS] [ID] ./myapp.pl minion job - ./myapp.pl minion job -t foo -S inactive - ./myapp.pl minion job -e foo -a '[23, "bar"]' - ./myapp.pl minion job -e foo -p 5 -q important - ./myapp.pl minion job -s - ./myapp.pl minion job -w -l 5 ./myapp.pl minion job 10023 + ./myapp.pl minion job -w + ./myapp.pl minion job -w 23 + ./myapp.pl minion job -s + ./myapp.pl minion job -q important -t foo -S inactive + ./myapp.pl minion job -e foo -a '[23, "bar"]' + ./myapp.pl minion job -e foo -P 10023 -P 10024 -p 5 -q important ./myapp.pl minion job -R -d 10 10023 ./myapp.pl minion job -r 10023 Options: - -A, --attempts <number> Number of times performing this job will be + -A, --attempts <number> Number of times performing this new job will be attempted, defaults to 1 -a, --args <JSON array> Arguments for new job in JSON format -d, --delay <seconds> Delay new job for this many seconds @@ -135,14 +108,17 @@ the value of MOJO_MODE/PLACK_ENV or "development" -o, --offset <number> Number of jobs/workers to skip when listing them, defaults to 0 + -P, --parent <id> One or more jobs the new job depends on -p, --priority <number> Priority of new job, defaults to 0 - -q, --queue <name> Queue to put job in, defaults to "default" + -q, --queue <name> Queue to put new job in, defaults to "default", or + list only jobs in this queue -R, --retry Retry job -r, --remove Remove job -S, --state <state> List only jobs in this state -s, --stats Show queue statistics -t, --task <name> List only jobs for this task - -w, --workers List workers instead of jobs + -w, --workers List workers instead of jobs, or show information + for a specific worker =head1 DESCRIPTION diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Command/minion/worker.pm new/Minion-5.08/lib/Minion/Command/minion/worker.pm --- old/Minion-5.03/lib/Minion/Command/minion/worker.pm 2016-02-24 12:02:24.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Command/minion/worker.pm 2016-05-05 04:21:11.000000000 +0200 @@ -21,6 +21,9 @@ local $SIG{INT} = local $SIG{TERM} = sub { $self->{finished}++ }; local $SIG{QUIT} = sub { ++$self->{finished} and kill 'KILL', keys %{$self->{jobs}} }; + local $SIG{TTIN} = sub { $self->{max}++ }; + local $SIG{TTOU} = sub { $self->{max}-- if $self->{max} > 0 }; + local $SIG{USR1} = sub { $self->{max} = 0 }; # Log fatal errors my $app = $self->app; @@ -114,6 +117,20 @@ Stop immediately without finishing the current jobs. +=head2 TTIN + +Increase the number of jobs to perform concurrently by one. + +=head2 TTOU + +Decrease the number of jobs to perform concurrently by one. + +=head2 USR1 + +Pause the worker by setting the number of jobs to perform concurrently to zero. +That means it will finish all current jobs, but not accept new ones, until the +number is increased again with L</"TTIN">. + =head1 ATTRIBUTES L<Minion::Command::minion::worker> inherits all attributes from diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Job.pm new/Minion-5.08/lib/Minion/Job.pm --- old/Minion-5.03/lib/Minion/Job.pm 2016-02-16 22:27:01.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Job.pm 2016-05-17 03:24:42.000000000 +0200 @@ -245,23 +245,35 @@ Number of times performing this job will be attempted. +=item children + + children => ['10026', '10027', '10028'] + +Jobs depending on this job. + =item created created => 784111777 -Time job was created. +Epoch time job was created. =item delayed delayed => 784111777 -Time job was delayed to. +Epoch time job was delayed to. =item finished finished => 784111777 -Time job was finished. +Epoch time job was finished. + +=item parents + + parents => ['10023', '10024', '10025'] + +Jobs this job depends on. =item priority @@ -285,7 +297,7 @@ retried => 784111777 -Time job has been retried. +Epoch time job has been retried. =item retries @@ -297,7 +309,7 @@ started => 784111777 -Time job was started. +Epoch time job was started. =item state diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion/Worker.pm new/Minion-5.08/lib/Minion/Worker.pm --- old/Minion-5.03/lib/Minion/Worker.pm 2016-02-05 21:57:33.000000000 +0100 +++ new/Minion-5.08/lib/Minion/Worker.pm 2016-04-29 16:09:25.000000000 +0200 @@ -143,7 +143,7 @@ notified => 784111777 -Last time worker sent a heartbeat. +Epoch time worker sent the last heartbeat. =item pid @@ -155,7 +155,7 @@ started => 784111777 -Time worker was started. +Epoch time worker was started. =back diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/lib/Minion.pm new/Minion-5.08/lib/Minion.pm --- old/Minion-5.03/lib/Minion.pm 2016-04-10 18:57:04.000000000 +0200 +++ new/Minion-5.08/lib/Minion.pm 2016-05-19 06:26:47.000000000 +0200 @@ -6,7 +6,6 @@ use Minion::Worker; use Mojo::Loader 'load_class'; use Mojo::Server; -use Mojo::URL; use Scalar::Util 'weaken'; has app => sub { Mojo::Server->new->build_app('Mojo::HelloWorld') }; @@ -16,7 +15,7 @@ has remove_after => 172800; has tasks => sub { {} }; -our $VERSION = '5.03'; +our $VERSION = '5.08'; sub add_task { ($_[0]->tasks->{$_[1]} = $_[2]) and return $_[0] } @@ -121,9 +120,9 @@ L<Minion> is a job queue for the L<Mojolicious|http://mojolicious.org> real-time web framework, with support for multiple named queues, priorities, delayed jobs, -job results, retries with backoff, statistics, distributed workers, parallel -processing, autoscaling, resource leak protection and multiple backends (such as -L<PostgreSQL|http://www.postgresql.org>). +job dependencies, job results, retries with backoff, statistics, distributed +workers, parallel processing, autoscaling, resource leak protection and multiple +backends (such as L<PostgreSQL|http://www.postgresql.org>). Job queues allow you to process time and/or computationally intensive tasks in background processes, outside of the request/response lifecycle. Among those @@ -276,8 +275,8 @@ $minion = $minion->remove_after(86400); Amount of time in seconds after which jobs that have reached the state -C<finished> will be removed automatically by L</"repair">, defaults to -C<172800> (2 days). +C<finished> and have no unresolved dependencies will be removed automatically by +L</"repair">, defaults to C<172800> (2 days). =head2 tasks @@ -333,6 +332,13 @@ Delay job for this many seconds (from now). +=item parents + + parents => [$id1, $id2, $id3] + +One or more existing jobs this job depends on, and that need to have +transitioned to the state C<finished> before it can be processed. + =item priority priority => 5 @@ -427,6 +433,14 @@ Number of workers that are currently processing a job. +=item delayed_jobs + + delayed_jobs => 100 + +Number of jobs in C<inactive> state that are scheduled to run at specific time +in the future or have unresolved dependencies. Note that this field is +EXPERIMENTAL and might change without warning! + =item failed_jobs failed_jobs => 100 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-5.03/t/pg.t new/Minion-5.08/t/pg.t --- old/Minion-5.03/t/pg.t 2016-04-10 19:00:37.000000000 +0200 +++ new/Minion-5.08/t/pg.t 2016-05-19 04:36:04.000000000 +0200 @@ -24,10 +24,11 @@ isa_ok $worker->minion->app, 'Mojolicious', 'has default application'; # Migrate up and down -is $minion->backend->pg->migrations->active, 8, 'active version is 8'; +is $minion->backend->pg->migrations->active, 10, 'active version is 10'; is $minion->backend->pg->migrations->migrate(0)->active, 0, 'active version is 0'; -is $minion->backend->pg->migrations->migrate->active, 8, 'active version is 8'; +is $minion->backend->pg->migrations->migrate->active, 10, + 'active version is 10'; # Register and unregister $worker->register; @@ -147,6 +148,7 @@ is $stats->{failed_jobs}, 0, 'no failed jobs'; is $stats->{finished_jobs}, 0, 'no finished jobs'; is $stats->{inactive_jobs}, 0, 'no inactive jobs'; +is $stats->{delayed_jobs}, 0, 'no delayed jobs'; $worker = $minion->worker->register; is $minion->stats->{inactive_workers}, 1, 'one inactive worker'; $minion->enqueue('fail'); @@ -180,6 +182,7 @@ is $stats->{failed_jobs}, 0, 'no failed jobs'; is $stats->{finished_jobs}, 3, 'three finished jobs'; is $stats->{inactive_jobs}, 0, 'no inactive jobs'; +is $stats->{delayed_jobs}, 0, 'no delayed jobs'; # List jobs $id = $minion->enqueue('add'); @@ -192,8 +195,10 @@ is $batch->[1]{task}, 'fail', 'right task'; is_deeply $batch->[1]{args}, [], 'right arguments'; is_deeply $batch->[1]{result}, ['works'], 'right result'; -is $batch->[1]{state}, 'finished', 'right state'; -is $batch->[1]{priority}, 0, 'right priority'; +is $batch->[1]{state}, 'finished', 'right state'; +is $batch->[1]{priority}, 0, 'right priority'; +is_deeply $batch->[1]{parents}, [], 'right parents'; +is_deeply $batch->[1]{children}, [], 'right children'; is $batch->[1]{retries}, 1, 'job has been retried'; like $batch->[1]{created}, qr/^[\d.]+$/, 'has created timestamp'; like $batch->[1]{delayed}, qr/^[\d.]+$/, 'has delayed timestamp'; @@ -215,6 +220,14 @@ is $batch->[0]{task}, 'add', 'right task'; is $batch->[0]{retries}, 0, 'job has not been retried'; ok !$batch->[1], 'no more results'; +$batch = $minion->backend->list_jobs(0, 10, {queue => 'default'}); +is $batch->[0]{queue}, 'default', 'right queue'; +is $batch->[1]{queue}, 'default', 'right queue'; +is $batch->[2]{queue}, 'default', 'right queue'; +is $batch->[3]{queue}, 'default', 'right queue'; +ok !$batch->[4], 'no more results'; +$batch = $minion->backend->list_jobs(0, 10, {queue => 'does_not_exist'}); +is_deeply $batch, [], 'no results'; $batch = $minion->backend->list_jobs(0, 1); is $batch->[0]{state}, 'inactive', 'right state'; is $batch->[0]{retries}, 0, 'job has not been retried'; @@ -332,6 +345,7 @@ # Delayed jobs $id = $minion->enqueue(add => [2, 1] => {delay => 100}); +is $minion->stats->{delayed_jobs}, 1, 'one delayed job'; is $worker->register->dequeue(0), undef, 'too early for job'; ok $minion->job($id)->info->{delayed} > time, 'delayed timestamp'; $minion->backend->pg->db->query( @@ -388,7 +402,7 @@ is $finished, 0, 'finished event has not been emitted'; my $result; $job->on(finished => sub { $result = pop }); -$job->finish('Everything is fine!'); +ok $job->finish('Everything is fine!'), 'job finished'; $job->perform; is $result, 'Everything is fine!', 'right result'; is $failed, 0, 'failed event has not been emitted'; @@ -582,6 +596,48 @@ 'right result'; $worker->unregister; +# Job dependencies +$worker = $minion->remove_after(0)->worker->register; +is $minion->repair->stats->{finished_jobs}, 0, 'no finished jobs'; +$id = $minion->enqueue('test'); +$id2 = $minion->enqueue('test'); +$id3 = $minion->enqueue(test => [] => {parents => [$id, $id2]}); +is $minion->stats->{delayed_jobs}, 1, 'one delayed job'; +$job = $worker->dequeue(0); +is $job->id, $id, 'right id'; +is_deeply $job->info->{children}, [$id3], 'right children'; +is_deeply $job->info->{parents}, [], 'right parents'; +$job2 = $worker->dequeue(0); +is $job2->id, $id2, 'right id'; +is_deeply $job2->info->{children}, [$id3], 'right children'; +is_deeply $job2->info->{parents}, [], 'right parents'; +ok !$worker->dequeue(0), 'parents are not ready yet'; +ok $job->finish, 'job finished'; +ok !$worker->dequeue(0), 'parents are not ready yet'; +ok $job2->fail, 'job failed'; +ok !$worker->dequeue(0), 'parents are not ready yet'; +ok $job2->retry, 'job retried'; +$job2 = $worker->dequeue(0); +is $job2->id, $id2, 'right id'; +ok $job2->finish, 'job finished'; +$job = $worker->dequeue(0); +is $job->id, $id3, 'right id'; +is_deeply $job->info->{children}, [], 'right children'; +is_deeply $job->info->{parents}, [$id, $id2], 'right parents'; +is $minion->stats->{finished_jobs}, 2, 'two finished jobs'; +is $minion->repair->stats->{finished_jobs}, 2, 'two finished jobs'; +ok $job->finish, 'job finished'; +is $minion->stats->{finished_jobs}, 3, 'three finished jobs'; +is $minion->repair->stats->{finished_jobs}, 0, 'no finished jobs'; +$id = $minion->enqueue(test => [] => {parents => [-1]}); +ok !$worker->dequeue(0), 'job with missing parent will never be ready'; +$minion->repair; +like $minion->job($id)->info->{finished}, qr/^[\d.]+$/, + 'has finished timestamp'; +is $minion->job($id)->info->{state}, 'failed', 'right state'; +is $minion->job($id)->info->{result}, 'Parent went away', 'right result'; +$worker->unregister; + # Clean up once we are done $pg->db->query('drop schema minion_test cascade');
