Hello community, here is the log from the commit of package perl-Minion for openSUSE:Factory checked in at 2017-06-29 15:14:22 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/perl-Minion (Old) and /work/SRC/openSUSE:Factory/.perl-Minion.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "perl-Minion" Thu Jun 29 15:14:22 2017 rev:26 rq:506348 version:7.01 Changes: -------- --- /work/SRC/openSUSE:Factory/perl-Minion/perl-Minion.changes 2017-06-09 15:57:57.698975480 +0200 +++ /work/SRC/openSUSE:Factory/.perl-Minion.new/perl-Minion.changes 2017-06-29 15:14:37.925152593 +0200 @@ -1,0 +2,27 @@ +Mon Jun 26 05:46:05 UTC 2017 - co...@suse.com + +- updated to 7.01 + see /usr/share/doc/packages/perl-Minion/Changes + + 7.01 2017-06-25 + - Added note methods to Minion::Job and Minion::Backend::Pg. + - Added notes option to enqueue methods in Minion and Minion::Backend::Pg. + - Added notes field to info method in Minion::Job and job_info method in + Minion::Backend::Pg. + - Improved performance of stats and lock methods in Minion::Backend::Pg with a + new index and other optimizations. (depesz) + - Improved benchmark script to be more consistent. (depesz) + +------------------------------------------------------------------- +Tue Jun 20 05:59:26 UTC 2017 - co...@suse.com + +- updated to 7.0 + see /usr/share/doc/packages/perl-Minion/Changes + + 7.0 2017-06-18 + - Added support for rate limiting and unique jobs. + - Added lock and unlock methods to Minion and Minion::Backend::Pg. + - Improved performance of Minion::Backend::Pg significantly with a new index + and other optimizations. + +------------------------------------------------------------------- Old: ---- Minion-6.06.tar.gz New: ---- Minion-7.01.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ perl-Minion.spec ++++++ --- /var/tmp/diff_new_pack.Fub7c0/_old 2017-06-29 15:14:41.652625657 +0200 +++ /var/tmp/diff_new_pack.Fub7c0/_new 2017-06-29 15:14:41.652625657 +0200 @@ -17,7 +17,7 @@ Name: perl-Minion -Version: 6.06 +Version: 7.01 Release: 0 %define cpan_name Minion Summary: Job queue @@ -37,10 +37,10 @@ %description Minion is a job queue for the at http://mojolicious.org real-time web framework, with support for multiple named queues, priorities, delayed -jobs, job dependencies, job results, retries with backoff, statistics, -distributed workers, parallel processing, autoscaling, remote control, -resource leak protection and multiple backends (such as at -http://www.postgresql.org). +jobs, job dependencies, job progress, job results, retries with backoff, +rate limiting, unique jobs, statistics, distributed workers, parallel +processing, autoscaling, remote control, resource leak protection and +multiple backends (such as at http://www.postgresql.org). Job queues allow you to process time and/or computationally intensive tasks in background processes, outside of the request/response lifecycle. Among ++++++ Minion-6.06.tar.gz -> Minion-7.01.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/Changes new/Minion-7.01/Changes --- old/Minion-6.06/Changes 2017-06-03 20:01:21.000000000 +0200 +++ new/Minion-7.01/Changes 2017-06-25 17:50:48.000000000 +0200 @@ -1,4 +1,19 @@ +7.01 2017-06-25 + - Added note methods to Minion::Job and Minion::Backend::Pg. + - Added notes option to enqueue methods in Minion and Minion::Backend::Pg. + - Added notes field to info method in Minion::Job and job_info method in + Minion::Backend::Pg. + - Improved performance of stats and lock methods in Minion::Backend::Pg with a + new index and other optimizations. (depesz) + - Improved benchmark script to be more consistent. (depesz) + +7.0 2017-06-18 + - Added support for rate limiting and unique jobs. + - Added lock and unlock methods to Minion and Minion::Backend::Pg. + - Improved performance of Minion::Backend::Pg significantly with a new index + and other optimizations. + 6.06 2017-06-03 - Added an example application to demonstrate how to integrate background jobs into well-structured Mojolicious applications. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/MANIFEST new/Minion-7.01/MANIFEST --- old/Minion-6.06/MANIFEST 2017-06-06 00:14:54.000000000 +0200 +++ new/Minion-7.01/MANIFEST 2017-06-25 20:57:36.000000000 +0200 @@ -24,6 +24,7 @@ MANIFEST.bak MANIFEST.SKIP README.md +t/backend.t t/commands.t t/pg.t t/pg_lite_app.t diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/META.json new/Minion-7.01/META.json --- old/Minion-6.06/META.json 2017-06-06 00:14:54.000000000 +0200 +++ new/Minion-7.01/META.json 2017-06-25 20:57:36.000000000 +0200 @@ -4,7 +4,7 @@ "Sebastian Riedel <s...@cpan.org>" ], "dynamic_config" : 0, - "generated_by" : "ExtUtils::MakeMaker version 7.28, CPAN::Meta::Converter version 2.150010", + "generated_by" : "ExtUtils::MakeMaker version 7.3, CPAN::Meta::Converter version 2.150010", "license" : [ "artistic_2" ], @@ -54,6 +54,6 @@ }, "x_IRC" : "irc://irc.perl.org/#mojo" }, - "version" : "6.06", + "version" : "7.01", "x_serialization_backend" : "JSON::PP version 2.94" } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/META.yml new/Minion-7.01/META.yml --- old/Minion-6.06/META.yml 2017-06-06 00:14:54.000000000 +0200 +++ new/Minion-7.01/META.yml 2017-06-25 20:57:36.000000000 +0200 @@ -7,7 +7,7 @@ configure_requires: ExtUtils::MakeMaker: '0' dynamic_config: 0 -generated_by: 'ExtUtils::MakeMaker version 7.28, CPAN::Meta::Converter version 2.150010' +generated_by: 'ExtUtils::MakeMaker version 7.3, CPAN::Meta::Converter version 2.150010' license: artistic_2 meta-spec: url: http://module-build.sourceforge.net/META-spec-v1.4.html @@ -27,5 +27,5 @@ homepage: http://mojolicious.org license: http://www.opensource.org/licenses/artistic-license-2.0 repository: https://github.com/kraih/minion.git -version: '6.06' +version: '7.01' x_serialization_backend: 'CPAN::Meta::YAML version 0.018' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/README.md new/Minion-7.01/README.md --- old/Minion-6.06/README.md 2017-04-03 20:12:28.000000000 +0200 +++ new/Minion-7.01/README.md 2017-06-25 20:09:02.000000000 +0200 @@ -3,9 +3,10 @@ A job queue for the [Mojolicious](http://mojolicious.org) real-time web framework, with support for multiple named queues, priorities, delayed jobs, - job results, retries with backoff, statistics, distributed workers, parallel - processing, autoscaling, resource leak protection and multiple backends (such - as [PostgreSQL](http://www.postgresql.org)). + job progress, job results, retries with backoff, rate limiting, unique jobs, + statistics, distributed workers, parallel processing, autoscaling, resource + leak protection and multiple backends (such as + [PostgreSQL](http://www.postgresql.org)). Job queues allow you to process time and/or computationally intensive tasks in background processes, outside of the request/response lifecycle. Among those diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/examples/minion_bench.pl new/Minion-7.01/examples/minion_bench.pl --- old/Minion-6.06/examples/minion_bench.pl 2016-10-23 21:49:55.000000000 +0200 +++ new/Minion-7.01/examples/minion_bench.pl 2017-06-25 20:01:34.000000000 +0200 @@ -7,9 +7,11 @@ my $DEQUEUE = 1000; my $REPETITIONS = 2; my $WORKERS = 4; +my $INFO = 100; my $STATS = 100; my $REPAIR = 100; -my $INFO = 100; +my $LOCK = 1000; +my $UNLOCK = 1000; # A benchmark script for comparing backends and evaluating the performance # impact of proposed changes @@ -20,11 +22,14 @@ # Enqueue say "Clean start with $ENQUEUE jobs"; +my @parents = map { $minion->enqueue('foo') } 1 .. 5; my $before = time; -$minion->enqueue($_ % 2 ? 'foo' : 'bar') for 1 .. $ENQUEUE; +$minion->enqueue($_ % 2 ? 'foo' : 'bar' => [] => {parents => \@parents}) + for 1 .. $ENQUEUE; my $elapsed = time - $before; my $avg = sprintf '%.3f', $ENQUEUE / $elapsed; say "Enqueued $ENQUEUE jobs in $elapsed seconds ($avg/s)"; +$minion->backend->pg->db->query('analyze minion_jobs'); # Dequeue sub dequeue { @@ -32,7 +37,7 @@ for (1 .. $WORKERS) { die "Couldn't fork: $!" unless defined(my $pid = fork); unless ($pid) { - my $worker = $minion->worker->register; + my $worker = $minion->repair->worker->register; say "$$ will finish $DEQUEUE jobs"; my $before = time; $worker->dequeue(0.5)->finish for 1 .. $DEQUEUE; @@ -55,6 +60,15 @@ } dequeue() for 1 .. $REPETITIONS; +# Job info +say "Requesting job info $INFO times"; +$before = time; +my $backend = $minion->backend; +$backend->job_info($_) for 1 .. $INFO; +$elapsed = time - $before; +$avg = sprintf '%.3f', $INFO / $elapsed; +say "Received job info $INFO times in $elapsed seconds ($avg/s)"; + # Stats say "Requesting stats $STATS times"; $before = time; @@ -71,11 +85,19 @@ $avg = sprintf '%.3f', $REPAIR / $elapsed; say "Repaired $REPAIR times in $elapsed seconds ($avg/s)"; -# Job info -say "Requesting job info $INFO times"; +# Lock +say "Acquiring locks $LOCK times"; $before = time; -my $backend = $minion->backend; -$backend->job_info($ENQUEUE) for 1 .. $INFO; +$minion->lock($_ % 2 ? 'foo' : 'bar', 3600, {limit => int($LOCK / 2)}) + for 1 .. $LOCK; $elapsed = time - $before; -$avg = sprintf '%.3f', $INFO / $elapsed; -say "Received job info $INFO times in $elapsed seconds ($avg/s)"; +$avg = sprintf '%.3f', $LOCK / $elapsed; +say "Acquired locks $LOCK times in $elapsed seconds ($avg/s)"; + +# Unlock +say "Releasing locks $UNLOCK times"; +$before = time; +$minion->unlock($_ % 2 ? 'foo' : 'bar') for 1 .. $UNLOCK; +$elapsed = time - $before; +$avg = sprintf '%.3f', $UNLOCK / $elapsed; +say "Releasing locks $UNLOCK times in $elapsed seconds ($avg/s)"; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/lib/Minion/Backend/Pg.pm new/Minion-7.01/lib/Minion/Backend/Pg.pm --- old/Minion-6.06/lib/Minion/Backend/Pg.pm 2017-06-02 23:03:51.000000000 +0200 +++ new/Minion-7.01/lib/Minion/Backend/Pg.pm 2017-06-25 20:34:58.000000000 +0200 @@ -12,7 +12,7 @@ my ($self, $command, $args, $ids) = (shift, shift, shift || [], shift || []); return !!$self->pg->db->query( q{update minion_workers set inbox = inbox || $1::jsonb - where (id = any($2) or $2 = '{}')}, {json => [[$command, @$args]]}, $ids + where (id = any ($2) or $2 = '{}')}, {json => [[$command, @$args]]}, $ids )->rows; } @@ -38,11 +38,12 @@ my $db = $self->pg->db; return $db->query( "insert into minion_jobs - (args, attempts, delayed, parents, priority, queue, task) - values (?, ?, (now() + (interval '1 second' * ?)), ?, ?, ?, ?) + (args, attempts, delayed, notes, parents, priority, queue, task) + values (?, ?, (now() + (interval '1 second' * ?)), ?, ?, ?, ?, ?) returning id", {json => $args}, $options->{attempts} // 1, - $options->{delay} // 0, $options->{parents} || [], - $options->{priority} // 0, $options->{queue} // 'default', $task + $options->{delay} // 0, {json => $options->{notes} || {}}, + $options->{parents} || [], $options->{priority} // 0, + $options->{queue} // 'default', $task )->hash->{id}; } @@ -52,11 +53,12 @@ sub job_info { shift->pg->db->query( 'select id, args, attempts, - array(select id from minion_jobs where j.id = any(parents)) as children, + array(select id from minion_jobs where parents @> ARRAY[j.id]) + as children, extract(epoch from created) as created, extract(epoch from delayed) as delayed, - extract(epoch from finished) as finished, parents, priority, queue, - result, extract(epoch from retried) as retried, retries, + extract(epoch from finished) as finished, notes, parents, priority, + queue, result, extract(epoch from retried) as retried, retries, extract(epoch from started) as started, state, task, worker from minion_jobs as j where id = ?', shift )->expand->hash; @@ -82,6 +84,20 @@ ->arrays->map(sub { $self->worker_info($_->[0]) })->to_array; } +sub lock { + my ($self, $name, $duration, $options) = (shift, shift, shift, shift // {}); + return !!$self->pg->db->query('select * from minion_lock(?, ?, ?)', + $name, $duration, $options->{limit} || 1)->array->[0]; +} + +sub note { + my ($self, $id, $key, $value) = @_; + return !!$self->pg->db->query( + 'update minion_jobs set notes = jsonb_set(notes, ?, ?, true) where id = ?', + [$key], {json => $value}, $id + )->rows; +} + sub new { my $self = shift->SUPER::new(pg => Mojo::Pg->new(@_)); @@ -142,28 +158,19 @@ )->hashes; $fail->each(sub { $self->fail_job(@$_{qw(id retries)}, 'Worker went away') }); - # Jobs with missing parents (can't be retried) - $db->query( - "update minion_jobs as j - set finished = now(), result = to_json('Parent went away'::text), - state = 'failed' - where parents <> '{}' and cardinality(parents) <> ( - select count(*) from minion_jobs where id = any (j.parents) - ) and state = 'inactive'" - ); - # Old jobs with no unresolved dependencies $db->query( "delete from minion_jobs as j where finished <= now() - interval '1 second' * ? and not exists ( select 1 from minion_jobs - where j.id = any(parents) and state <> 'finished' + where parents @> ARRAY[j.id] and state != 'finished' ) and state = 'finished'", $minion->remove_after ); } sub reset { - shift->pg->db->query('truncate minion_jobs, minion_workers restart identity'); + shift->pg->db->query( + 'truncate minion_jobs, minion_locks, minion_workers restart identity'); } sub retry_job { @@ -184,25 +191,32 @@ my $self = shift; my $stats = $self->pg->db->query( - "select 'enqueued_jobs', case when is_called then last_value else 0 end - from minion_jobs_id_seq - union all - select state::text || '_jobs', count(*) from minion_jobs group by state - union all - select 'delayed_jobs', count(*) from minion_jobs - where (delayed > now() or parents <> '{}') and state = 'inactive' - union all - select 'inactive_workers', count(*) from minion_workers - union all - select 'active_workers', count(distinct worker) from minion_jobs - where state = 'active'" - )->arrays->reduce(sub { $a->{$b->[0]} = $b->[1]; $a }, {}); + "select count(*) filter (where state = 'inactive') as inactive_jobs, + count(*) filter (where state = 'active') as active_jobs, + count(*) filter (where state = 'failed') as failed_jobs, + count(*) filter (where state = 'finished') as finished_jobs, + count(*) filter (where state = 'inactive' + and (delayed > now() or parents != '{}')) as delayed_jobs, + count(distinct worker) filter (where state = 'active') as active_workers, + (select case when is_called then last_value else 0 end + from minion_jobs_id_seq) as enqueued_jobs, + (select count(*) from minion_workers) as inactive_workers + from minion_jobs" + )->hash; $stats->{inactive_workers} -= $stats->{active_workers}; - $stats->{"${_}_jobs"} ||= 0 for qw(inactive active failed finished); return $stats; } +sub unlock { + !!shift->pg->db->query( + 'delete from minion_locks where id = ( + select id from minion_locks + where expires > now() and name = ? order by expires limit 1 + ) returning 1', shift + )->rows; +} + sub unregister_worker { shift->pg->db->query('delete from minion_workers where id = ?', shift); } @@ -226,10 +240,11 @@ set started = now(), state = 'active', worker = ? where id = ( select id from minion_jobs as j - where delayed <= now() and (parents = '{}' or cardinality(parents) = ( - select count(*) from minion_jobs - where id = any(j.parents) and state = 'finished' - )) and queue = any(?) and state = 'inactive' and task = any(?) + where delayed <= now() and (parents = '{}' or not exists ( + select 1 from minion_jobs + where id = any (j.parents) + and state in ('inactive', 'active', 'failed') + )) and queue = any (?) and state = 'inactive' and task = any (?) order by priority desc, id limit 1 for update skip locked @@ -377,6 +392,12 @@ Delay job for this many seconds (from now), defaults to C<0>. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job. + =item parents parents => [$id1, $id2, $id3] @@ -470,6 +491,12 @@ Epoch time job was finished. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job. + =item parents parents => ['10023', '10024', '10025'] @@ -569,6 +596,33 @@ Returns the same information as L</"worker_info"> but in batches. +=head2 lock + + my $bool = $backend->lock('foo', 3600); + my $bool = $backend->lock('foo', 3600, {limit => 20}); + +Try to acquire a named lock that will expire automatically after the given +amount of time in seconds. + +These options are currently available: + +=over 2 + +=item limit + + limit => 20 + +Number of shared locks with the same name that can be active at the same time, +defaults to C<1>. + +=back + +=head2 note + + my $bool = $backend->note($job_id, foo => 'bar'); + +Change a metadata field for a job. + =head2 new my $backend = Minion::Backend::Pg->new('postgresql://postgres@/test'); @@ -715,6 +769,12 @@ =back +=head2 unlock + + my $bool = $backend->unlock('foo'); + +Release a named lock. + =head2 unregister_worker $backend->unregister_worker($worker_id); @@ -860,18 +920,48 @@ drop function if exists minion_jobs_notify_workers(); -- 10 up -alter table minion_jobs add column parents bigint[] default '{}'; +alter table minion_jobs add column parents bigint[] not null default '{}'; -- 11 up create index on minion_jobs (state, priority desc, id); -- 12 up alter table minion_workers add column inbox jsonb - check(jsonb_typeof(inbox) = 'array') default '[]'; - --- 13 up -create index on minion_jobs using gin (parents); + check(jsonb_typeof(inbox) = 'array') not null default '[]'; -- 15 up alter table minion_workers add column status jsonb - check(jsonb_typeof(status) = 'object') default '{}'; + check(jsonb_typeof(status) = 'object') not null default '{}'; + +-- 16 up +create index on minion_jobs using gin (parents); +create table if not exists minion_locks ( + id bigserial not null primary key, + name text not null, + expires timestamp with time zone not null +); +create function minion_lock(text, int, int) returns bool as $$ +declare + new_expires timestamp with time zone = now() + (interval '1 second' * $2); +begin + delete from minion_locks where expires < now(); + lock table minion_locks in exclusive mode; + if (select count(*) >= $3 from minion_locks where name = $1) then + return false; + end if; + if new_expires > now() then + insert into minion_locks (name, expires) values ($1, new_expires); + end if; + return true; +end; +$$ language plpgsql; + +-- 16 down +drop function if exists minion_lock(text, int, int); +drop table if exists minion_locks; + +-- 17 up +alter table minion_jobs add column notes jsonb + check(jsonb_typeof(notes) = 'object') not null default '{}'; +alter table minion_locks set unlogged; +create index on minion_locks (name, expires); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/lib/Minion/Backend.pm new/Minion-7.01/lib/Minion/Backend.pm --- old/Minion-6.06/lib/Minion/Backend.pm 2017-04-03 15:01:11.000000000 +0200 +++ new/Minion-7.01/lib/Minion/Backend.pm 2017-06-25 19:45:15.000000000 +0200 @@ -13,6 +13,8 @@ sub job_info { croak 'Method "job_info" not implemented by subclass' } sub list_jobs { croak 'Method "list_jobs" not implemented by subclass' } sub list_workers { croak 'Method "list_workers" not implemented by subclass' } +sub lock { croak 'Method "lock" not implemented by subclass' } +sub note { croak 'Method "note" not implemented by subclass' } sub receive { croak 'Method "receive" not implemented by subclass' } sub register_worker { @@ -24,6 +26,7 @@ sub reset { croak 'Method "reset" not implemented by subclass' } sub retry_job { croak 'Method "retry_job" not implemented by subclass' } sub stats { croak 'Method "stats" not implemented by subclass' } +sub unlock { croak 'Method "unlock" not implemented by subclass' } sub unregister_worker { croak 'Method "unregister_worker" not implemented by subclass'; @@ -52,6 +55,8 @@ sub job_info {...} sub list_jobs {...} sub list_workers {...} + sub lock {...} + sub note {...} sub receive {...} sub register_worker {...} sub remove_job {...} @@ -59,6 +64,7 @@ sub reset {...} sub retry_job {...} sub stats {...} + sub unlock {...} sub unregister_worker {...} sub worker_info {...} @@ -167,6 +173,12 @@ Delay job for this many seconds (from now), defaults to C<0>. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job. + =item parents parents => [$id1, $id2, $id3] @@ -262,6 +274,12 @@ Epoch time job was finished. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job. + =item parents parents => ['10023', '10024', '10025'] @@ -363,6 +381,33 @@ Returns the same information as L</"worker_info"> but in batches. Meant to be overloaded in a subclass. +=head2 lock + + my $bool = $backend->lock('foo', 3600); + my $bool = $backend->lock('foo', 3600, {limit => 20}); + +Try to acquire a named lock that will expire automatically after the given +amount of time in seconds. + +These options are currently available: + +=over 2 + +=item limit + + limit => 20 + +Number of shared locks with the same name that can be active at the same time, +defaults to C<1>. + +=back + +=head2 note + + $backend->note($job_id, foo => 'bar'); + +Change a metadata field for a job. + =head2 receive my $commands = $backend->receive($worker_id); @@ -499,6 +544,12 @@ =back +=head2 unlock + + my $bool = $backend->unlock('foo'); + +Release a named lock. + =head2 unregister_worker $backend->unregister_worker($worker_id); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/lib/Minion/Job.pm new/Minion-7.01/lib/Minion/Job.pm --- old/Minion-6.06/lib/Minion/Job.pm 2017-04-02 19:05:59.000000000 +0200 +++ new/Minion-7.01/lib/Minion/Job.pm 2017-06-25 20:14:47.000000000 +0200 @@ -30,7 +30,7 @@ return 1; } -sub stop { kill 'KILL', shift->{pid} } +sub note { $_[0]->minion->backend->note($_[0]->id, @_[1, 2]) } sub perform { my $self = shift; @@ -66,6 +66,8 @@ POSIX::_exit(0); } +sub stop { kill 'KILL', shift->{pid} } + 1; =encoding utf8 @@ -230,6 +232,9 @@ # Check job state my $state = $job->info->{state}; + # Get job metadata + my $progress = $job->info->{notes}{progress}; + # Get job result my $result = $job->info->{result}; @@ -273,6 +278,12 @@ Epoch time job was finished. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job. + =item parents parents => ['10023', '10024', '10025'] @@ -341,6 +352,21 @@ Check if job performed with L</"start"> is finished. +=head2 note + + my $bool = $job->note(foo => 'bar'); + +Change a metadata field for this job. The new value will get serialized by +L<Minion/"backend"> (often with L<Mojo::JSON>), so you shouldn't send objects +and be careful with binary data, nested data structures with hash and array +references are fine though. + + # Share progress information + $job->note(progress => 95); + + # Share stats + $job->note(stats => {utime => '0.012628', stime => '0.002429'}); + =head2 perform $job->perform; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/lib/Minion.pm new/Minion-7.01/lib/Minion.pm --- old/Minion-6.06/lib/Minion.pm 2017-06-03 20:12:28.000000000 +0200 +++ new/Minion-7.01/lib/Minion.pm 2017-06-25 20:08:23.000000000 +0200 @@ -16,7 +16,7 @@ has remove_after => 172800; has tasks => sub { {} }; -our $VERSION = '6.06'; +our $VERSION = '7.01'; sub add_task { ($_[0]->tasks->{$_[1]} = $_[2]) and return $_[0] } @@ -40,6 +40,8 @@ ); } +sub lock { shift->backend->lock(@_) } + sub new { my $self = shift->SUPER::new; @@ -63,7 +65,8 @@ sub repair { shift->_delegate('repair') } sub reset { shift->_delegate('reset') } -sub stats { shift->backend->stats } +sub stats { shift->backend->stats } +sub unlock { shift->backend->unlock(@_) } sub worker { my $self = shift; @@ -125,10 +128,10 @@ L<Minion> is a job queue for the L<Mojolicious|http://mojolicious.org> real-time web framework, with support for multiple named queues, priorities, delayed jobs, -job dependencies, job results, retries with backoff, statistics, distributed -workers, parallel processing, autoscaling, remote control, resource leak -protection and multiple backends (such as -L<PostgreSQL|http://www.postgresql.org>). +job dependencies, job progress, job results, retries with backoff, rate +limiting, unique jobs, statistics, distributed workers, parallel processing, +autoscaling, remote control, resource leak protection and multiple backends +(such as L<PostgreSQL|http://www.postgresql.org>). Job queues allow you to process time and/or computationally intensive tasks in background processes, outside of the request/response lifecycle. Among those @@ -361,6 +364,15 @@ Delay job for this many seconds (from now), defaults to C<0>. +=item notes + + notes => {foo => 'bar', baz => [1, 2, 3]} + +Hash reference with arbitrary metadata for this job that gets serialized by the +L</"backend"> (often with L<Mojo::JSON>), so you shouldn't send objects and be +careful with binary data, nested data structures with hash and array references +are fine though. + =item parents parents => [$id1, $id2, $id3] @@ -392,9 +404,59 @@ # Check job state my $state = $minion->job($id)->info->{state}; + # Get job metadata + my $progress = $minion->$job($id)->info->{notes}{progress}; + # Get job result my $result = $minion->job($id)->info->{result}; +=head2 lock + + my $bool = $minion->lock('foo', 3600); + my $bool = $minion->lock('foo', 3600, {limit => 20}); + +Try to acquire a named lock that will expire automatically after the given +amount of time in seconds. You can release the lock manually with L</"unlock"> +to limit concurrency, or let it expire for rate limiting. + + # Only one job should run at a time (unique job) + $minion->add_task(do_unique_stuff => sub { + my ($job, @args) = @_; + return $job->finish('Previous job is still active') + unless $minion->lock('fragile_backend_service', 7200); + ... + $minion->unlock('fragile_backend_service'); + }); + + # Only five jobs should run at a time and we wait for our turn + $minion->add_task(do_concurrent_stuff => sub { + my ($job, @args) = @_; + sleep 1 until $minion->lock('some_web_service', 60, {limit => 5}); + ... + $minion->unlock('some_web_service'); + }); + + # Only a hundred jobs should run per hour and we try again later if necessary + $minion->add_task(do_rate_limited_stuff => sub { + my ($job, @args) = @_; + return $job->retry({delay => 3600}) + unless $minion->lock('another_web_service', 3600, {limit => 100}); + ... + }); + +These options are currently available: + +=over 2 + +=item limit + + limit => 20 + +Number of shared locks with the same name that can be active at the same time, +defaults to C<1>. + +=back + =head2 new my $minion = Minion->new(Pg => 'postgresql://postgres@/test'); @@ -503,6 +565,12 @@ =back +=head2 unlock + + my $bool = $minion->unlock('foo'); + +Release a named lock that has been previously acquired with L</"lock">. + =head2 worker my $worker = $minion->worker; @@ -553,6 +621,8 @@ Brian Medley +Hubert "depesz" Lubaczewski + Joel Berger Paul Williams diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/t/backend.t new/Minion-7.01/t/backend.t --- old/Minion-6.06/t/backend.t 1970-01-01 01:00:00.000000000 +0100 +++ new/Minion-7.01/t/backend.t 2017-06-25 17:50:00.000000000 +0200 @@ -0,0 +1,50 @@ +use Mojo::Base -strict; + +use Test::More; +use Minion::Backend; + +# Abstract methods +eval { Minion::Backend->broadcast }; +like $@, qr/Method "broadcast" not implemented by subclass/, 'right error'; +eval { Minion::Backend->dequeue }; +like $@, qr/Method "dequeue" not implemented by subclass/, 'right error'; +eval { Minion::Backend->enqueue }; +like $@, qr/Method "enqueue" not implemented by subclass/, 'right error'; +eval { Minion::Backend->fail_job }; +like $@, qr/Method "fail_job" not implemented by subclass/, 'right error'; +eval { Minion::Backend->finish_job }; +like $@, qr/Method "finish_job" not implemented by subclass/, 'right error'; +eval { Minion::Backend->job_info }; +like $@, qr/Method "job_info" not implemented by subclass/, 'right error'; +eval { Minion::Backend->list_jobs }; +like $@, qr/Method "list_jobs" not implemented by subclass/, 'right error'; +eval { Minion::Backend->list_workers }; +like $@, qr/Method "list_workers" not implemented by subclass/, 'right error'; +eval { Minion::Backend->lock }; +like $@, qr/Method "lock" not implemented by subclass/, 'right error'; +eval { Minion::Backend->note }; +like $@, qr/Method "note" not implemented by subclass/, 'right error'; +eval { Minion::Backend->receive }; +like $@, qr/Method "receive" not implemented by subclass/, 'right error'; +eval { Minion::Backend->register_worker }; +like $@, qr/Method "register_worker" not implemented by subclass/, + 'right error'; +eval { Minion::Backend->remove_job }; +like $@, qr/Method "remove_job" not implemented by subclass/, 'right error'; +eval { Minion::Backend->repair }; +like $@, qr/Method "repair" not implemented by subclass/, 'right error'; +eval { Minion::Backend->reset }; +like $@, qr/Method "reset" not implemented by subclass/, 'right error'; +eval { Minion::Backend->retry_job }; +like $@, qr/Method "retry_job" not implemented by subclass/, 'right error'; +eval { Minion::Backend->stats }; +like $@, qr/Method "stats" not implemented by subclass/, 'right error'; +eval { Minion::Backend->unlock }; +like $@, qr/Method "unlock" not implemented by subclass/, 'right error'; +eval { Minion::Backend->unregister_worker }; +like $@, qr/Method "unregister_worker" not implemented by subclass/, + 'right error'; +eval { Minion::Backend->worker_info }; +like $@, qr/Method "worker_info" not implemented by subclass/, 'right error'; + +done_testing(); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/Minion-6.06/t/pg.t new/Minion-7.01/t/pg.t --- old/Minion-6.06/t/pg.t 2017-04-10 20:27:14.000000000 +0200 +++ new/Minion-7.01/t/pg.t 2017-06-25 19:45:35.000000000 +0200 @@ -24,11 +24,11 @@ isa_ok $worker->minion->app, 'Mojolicious', 'has default application'; # Migrate up and down -is $minion->backend->pg->migrations->active, 15, 'active version is 15'; +is $minion->backend->pg->migrations->active, 17, 'active version is 17'; is $minion->backend->pg->migrations->migrate(0)->active, 0, 'active version is 0'; -is $minion->backend->pg->migrations->migrate->active, 15, - 'active version is 15'; +is $minion->backend->pg->migrations->migrate->active, 17, + 'active version is 17'; # Register and unregister $worker->register; @@ -130,11 +130,43 @@ $worker->unregister; $worker2->unregister; +# Exclusive lock +ok $minion->lock('foo', 3600), 'locked'; +ok !$minion->lock('foo', 3600), 'not locked again'; +ok $minion->unlock('foo'), 'unlocked'; +ok !$minion->unlock('foo'), 'not unlocked again'; +ok $minion->lock('foo', -3600), 'locked'; +ok $minion->lock('foo', 3600), 'locked again'; +ok !$minion->lock('foo', -3600), 'not locked again'; +ok !$minion->lock('foo', 3600), 'not locked again'; +ok $minion->unlock('foo'), 'unlocked'; +ok !$minion->unlock('foo'), 'not unlocked again'; +ok $minion->lock('yada', 3600, {limit => 1}), 'locked'; +ok !$minion->lock('yada', 3600, {limit => 1}), 'not locked again'; + +# Shared lock +ok $minion->lock('bar', 3600, {limit => 3}), 'locked'; +ok $minion->lock('bar', 3600, {limit => 3}), 'locked again'; +ok $minion->lock('bar', -3600, {limit => 3}), 'locked again'; +ok $minion->lock('bar', 3600, {limit => 3}), 'locked again'; +ok !$minion->lock('bar', 3600, {limit => 2}), 'not locked again'; +ok $minion->lock('baz', 3600, {limit => 3}), 'locked'; +ok $minion->unlock('bar'), 'unlocked'; +ok $minion->lock('bar', 3600, {limit => 3}), 'locked again'; +ok $minion->unlock('bar'), 'unlocked again'; +ok $minion->unlock('bar'), 'unlocked again'; +ok $minion->unlock('bar'), 'unlocked again'; +ok !$minion->unlock('bar'), 'not unlocked again'; +ok $minion->unlock('baz'), 'unlocked'; +ok !$minion->unlock('baz'), 'not unlocked again'; + # Reset $minion->reset->repair; ok !$minion->backend->pg->db->query( 'select count(id) as count from minion_jobs')->hash->{count}, 'no jobs'; ok !$minion->backend->pg->db->query( + 'select count(id) as count from minion_locks')->hash->{count}, 'no locks'; +ok !$minion->backend->pg->db->query( 'select count(id) as count from minion_workers')->hash->{count}, 'no workers'; # Stats @@ -201,6 +233,7 @@ like $batch->[0]{created}, qr/^[\d.]+$/, 'has created timestamp'; is $batch->[1]{task}, 'fail', 'right task'; is_deeply $batch->[1]{args}, [], 'right arguments'; +is_deeply $batch->[1]{notes}, {}, 'right metadata'; is_deeply $batch->[1]{result}, ['works'], 'right result'; is $batch->[1]{state}, 'finished', 'right state'; is $batch->[1]{priority}, 0, 'right priority'; @@ -475,13 +508,28 @@ $minion->add_task( nested => sub { my ($job, $hash, $array) = @_; + $job->note(bar => {baz => [1, 2, 3]}); + $job->note(baz => 'yada'); $job->finish([{23 => $hash->{first}[0]{second} x $array->[0][0]}]); } ); -$minion->enqueue(nested => [{first => [{second => 'test'}]}, [[3]]]); +$minion->enqueue( + 'nested', + [{first => [{second => 'test'}]}, [[3]]], + {notes => {foo => [4, 5, 6]}} +); $job = $worker->register->dequeue(0); $job->perform; is $job->info->{state}, 'finished', 'right state'; +ok $job->note(yada => ['works']), 'added metadata'; +ok !$minion->backend->note(-1, yada => ['failed']), 'not added metadata'; +my $notes = { + foo => [4, 5, 6], + bar => {baz => [1, 2, 3]}, + baz => 'yada', + yada => ['works'] +}; +is_deeply $job->info->{notes}, $notes, 'right metadata'; is_deeply $job->info->{result}, [{23 => 'testtesttest'}], 'right structure'; $worker->unregister; @@ -648,12 +696,9 @@ is $minion->stats->{finished_jobs}, 3, 'three finished jobs'; is $minion->repair->stats->{finished_jobs}, 0, 'no finished jobs'; $id = $minion->enqueue(test => [] => {parents => [-1]}); -ok !$worker->dequeue(0), 'job with missing parent will never be ready'; -$minion->repair; -like $minion->job($id)->info->{finished}, qr/^[\d.]+$/, - 'has finished timestamp'; -is $minion->job($id)->info->{state}, 'failed', 'right state'; -is $minion->job($id)->info->{result}, 'Parent went away', 'right result'; +$job = $worker->dequeue(0); +is $job->id, $id, 'right id'; +ok $job->finish, 'job finished'; $worker->unregister; # Worker remote control commands
