This gives "lei import", "lei tag", and similar commands the ability to use URLs recognized by our PSGI frontend directly.
This is more convenient than an equivalent shell pipeline since "set -o pipefail" is not portable and errors may be lost. --- MANIFEST | 1 + lib/PublicInbox/LeiInput.pm | 76 +++++++++++++++++++++++++++++++++++++ t/lei-import-http.t | 43 +++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 t/lei-import-http.t diff --git a/MANIFEST b/MANIFEST index 79d393c5..ce824fcf 100644 --- a/MANIFEST +++ b/MANIFEST @@ -394,6 +394,7 @@ t/kqnotify.t t/lei-convert.t t/lei-daemon.t t/lei-externals.t +t/lei-import-http.t t/lei-import-imap.t t/lei-import-maildir.t t/lei-import-nntp.t diff --git a/lib/PublicInbox/LeiInput.pm b/lib/PublicInbox/LeiInput.pm index d11d23d4..785e607d 100644 --- a/lib/PublicInbox/LeiInput.pm +++ b/lib/PublicInbox/LeiInput.pm @@ -6,6 +6,7 @@ package PublicInbox::LeiInput; use strict; use v5.10.1; use PublicInbox::DS; +use PublicInbox::Spawn qw(which popen_rd); # JMAP RFC 8621 4.1.1 # https://www.iana.org/assignments/imap-jmap-keywords/imap-jmap-keywords.xhtml @@ -77,6 +78,32 @@ error reading $name: $! } } +# handles mboxrd endpoints described in Documentation/design_notes.txt +sub handle_http_input ($$@) { + my ($self, $url, @args) = @_; + my $lei = $self->{lei} or die 'BUG: {lei} missing'; + my $curl_opt = delete $self->{"-curl-$url"} or + die("BUG: $url curl options not prepared"); + my $uri = pop @$curl_opt; + my $curl = PublicInbox::LeiCurl->new($lei, $self->{curl}) or return; + push @$curl, '-s', @$curl_opt; + my $cmd = $curl->for_uri($lei, $uri); + $lei->qerr("# $cmd"); + my $rdr = { 2 => $lei->{2}, pgid => 0 }; + my ($fh, $pid) = popen_rd($cmd, undef, $rdr); + grep(/\A--compressed\z/, @$curl) or + $fh = IO::Uncompress::Gunzip->new($fh, MultiStream => 1); + eval { + PublicInbox::MboxReader->mboxrd($fh, + $self->can('input_mbox_cb'), + $self, @args); + }; + my $err = $@; + waitpid($pid, 0); + $? || $err and + $lei->child_error($? || 1, "@$cmd failed".$err ? " $err" : ''); +} + sub input_path_url { my ($self, $input, @args) = @_; my $lei = $self->{lei}; @@ -92,6 +119,9 @@ sub input_path_url { $self->can('input_net_cb'), $self, @args); return; + } elsif ($input =~ m!\Ahttps?://!i) { + handle_http_input($self, $input, @args); + return; } if ($input =~ s!\A([a-z0-9]+):!!i) { $ifmt = lc($1); @@ -129,6 +159,50 @@ EOM } } +sub bad_http ($$;$) { + my ($lei, $url, $alt) = @_; + my $x = $alt ? "did you mean <$alt>?" : 'download and import manually'; + $lei->fail("E: <$url> not recognized, $x"); +} + +sub prepare_http_input ($$$) { + my ($self, $lei, $url) = @_; + require URI; + require PublicInbox::MboxReader; + require PublicInbox::LeiCurl; + require IO::Uncompress::Gunzip; + $self->{curl} //= which('curl') or + return $lei->fail("curl missing for <$url>"); + my $uri = URI->new($url); + my $path = $uri->path; + my %qf = $uri->query_form; + my @curl_opt; + if ($path =~ m!/(?:t\.mbox\.gz|all\.mbox\.gz)\z!) { + # OK + } elsif ($path =~ m!/raw\z!) { + push @curl_opt, '--compressed'; + # convert search query to mboxrd request since they require POST + # this is only intended for PublicInbox::WWW, and will false-positive + # on many other search engines... oh well + } elsif (defined $qf{'q'}) { + $qf{x} = 'm'; + $uri->query_form(\%qf); + push @curl_opt, '-d', ''; + $$uri ne $url and $lei->qerr(<<""); +# <$url> rewritten to <$$uri> with HTTP POST + + # try to provide hints for /$INBOX/$MSGID/T/ and /$INBOX/ + } elsif ($path =~ s!/[tT]/\z!/t.mbox.gz! || + $path =~ s!/t\.atom\z!/t.mbox.gz! || + $path =~ s!/([^/]+\@[^/]+)/\z!/$1/raw!) { + $uri->path($path); + return bad_http($lei, $url, $$uri); + } else { + return bad_http($lei, $url); + } + $self->{"-curl-$url"} = [ @curl_opt, $uri ]; # for handle_http_input +} + sub prepare_inputs { # returns undef on error my ($self, $lei, $inputs) = @_; my $in_fmt = $lei->{opt}->{'in-format'}; @@ -156,6 +230,8 @@ sub prepare_inputs { # returns undef on error push @{$sync->{no}}, $input; } } + } elsif ($input_path =~ m!\Ahttps?://!i) { + prepare_http_input($self, $lei, $input_path) or return; } elsif ($input_path =~ s/\A([a-z0-9]+)://is) { my $ifmt = lc $1; if (($in_fmt // $ifmt) ne $ifmt) { diff --git a/t/lei-import-http.t b/t/lei-import-http.t new file mode 100644 index 00000000..35cbf369 --- /dev/null +++ b/t/lei-import-http.t @@ -0,0 +1,43 @@ +#!perl -w +# Copyright (C) 2021 all contributors <[email protected]> +# License: AGPL-3.0+ <https://www.gnu.org/licenses/agpl-3.0.txt> +use strict; use v5.10.1; use PublicInbox::TestCommon; +use PublicInbox::Spawn qw(which); +require_mods(qw(lei -httpd)); +which('curl') or plan skip_all => "curl required for $0"; +my ($ro_home, $cfg_path) = setup_public_inboxes; +my ($tmpdir, $for_destroy) = tmpdir; +my $sock = tcp_server; +my $cmd = [ '-httpd', '-W0', "--stdout=$tmpdir/1", "--stderr=$tmpdir/2" ]; +my $env = { PI_CONFIG => $cfg_path }; +my $td = start_script($cmd, $env, { 3 => $sock }) or BAIL_OUT("-httpd $?"); +my $host_port = tcp_host_port($sock); +undef $sock; +test_lei({ tmpdir => $tmpdir }, sub { + my $url = "http://$host_port/t2";; + for my $p (qw(bogus@x/t.mbox.gz bogus@x/raw ?q=noresultever)) { + ok(!lei('import', "$url/$p"), "/$p fails properly"); + } + for my $p (qw(/ /T/ /t/ /t.atom)) { + ok(!lei('import', "$url/m\@example$p"), "/$p fails"); + like($lei_err, qr/did you mean/, "gave hint for $p"); + } + lei_ok 'import', "$url/testmessage\@example.com/raw"; + lei_ok 'q', 'm:[email protected]'; + my $res = json_utf8->decode($lei_out); + is($res->[0]->{'m'}, '[email protected]', 'imported raw') + or diag explain($res); + + lei_ok 'import', "$url/qp\@example.com/t.mbox.gz"; + lei_ok 'q', 'm:[email protected]'; + $res = json_utf8->decode($lei_out); + is($res->[0]->{'m'}, '[email protected]', 'imported t.mbox.gz') + or diag explain($res); + + lei_ok 'import', "$url/?q=s:boolean"; + lei_ok 'q', 'm:20180720072141.GA15957@example'; + $res = json_utf8->decode($lei_out); + is($res->[0]->{'m'}, '20180720072141.GA15957@example', + 'imported search result') or diag explain($res); +}); +done_testing; -- unsubscribe: one-click, see List-Unsubscribe header archive: https://public-inbox.org/meta/
