Hello Eric, Hello Konstantin, On 20.06.23 04:37, Eric Wong wrote: > Konstantin Ryabitsev <[email protected]> wrote: >> On Wed, Jun 14, 2023 at 11:50:15PM +0000, Eric Wong wrote: >>> affects WWW and NNTP, but not IMAP/POP3. I'm not sure if I want >>> to reintroduce header injection in case there's some conflict >>> with DKIM or other signature mechanisms[1] >> >> I don't think we need to worry about it if we pick a header that's almost >> certain to not be included in the default DKIM signature set. >> X-Originally-Archived-At: or some other header is guaranteed to never be >> signed. > > *shrug* I'm not sure how useful this is, actually.
Thanks for the patch, Eric! Konstantin, would this work for you? Is there something I could help with? Thanks, Ahmad > > -----------8<--------- > Subject: [PATCH] support publicinbox.$FOO.appendHeader in read-only endpoints > > This may be used to inject arbitrary headers in the raw message > from any read-only endpoints. This may be useful for mirrors to > indicate they're mirroring another source and can't edit/remove > messages easily. > > This is set per-inbox and supports multiple key:value pairs: > > [publicinbox "foo"] > appendheader = KEY1:VALUE1 > appendheader = KEY2:VALUE2 > > No variable expansion is currently supported as it's unclear if > it's necessary (e.g. for Message-ID in URLs). > > Link: > https://public-inbox.org/meta/20230615-focal-erosion-poop-df8246@meerkat/ > --- > lib/PublicInbox/Config.pm | 2 +- > lib/PublicInbox/Eml.pm | 20 +++++++++++++++ > lib/PublicInbox/IMAP.pm | 17 ++++++++++--- > lib/PublicInbox/Inbox.pm | 8 ++++++ > lib/PublicInbox/Mbox.pm | 8 +++--- > lib/PublicInbox/NNTP.pm | 2 ++ > lib/PublicInbox/POP3.pm | 2 +- > t/netd.t | 53 ++++++++++++++++++++++++++++++++------- > 8 files changed, 94 insertions(+), 18 deletions(-) > > diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm > index 2f1b4122..c3ec1793 100644 > --- a/lib/PublicInbox/Config.pm > +++ b/lib/PublicInbox/Config.pm > @@ -457,7 +457,7 @@ sub _fill_ibx { > # more things to encourage decentralization > for my $k (qw(address altid nntpmirror imapmirror > coderepo hide listid url > - infourl watchheader > + infourl watchheader appendheader > nntpserver imapserver pop3server)) { > my $v = $self->{"$pfx.$k"} // next; > $ibx->{$k} = _array($v); > diff --git a/lib/PublicInbox/Eml.pm b/lib/PublicInbox/Eml.pm > index 8b999e1a..3f7e27e0 100644 > --- a/lib/PublicInbox/Eml.pm > +++ b/lib/PublicInbox/Eml.pm > @@ -392,6 +392,26 @@ sub header_str_set { > header_set($self, $name, @vals); > } > > +# only used for publicinbox.$NAME.appendHeader > +sub header_append { > + my ($self, $pfx, @vals) = @_; > + $pfx .= ': '; > + my $len = 78 - length($pfx); > + @vals = map {; > + utf8::encode(my $v = $_); # to bytes, support SMTPUTF8 > + # folding differs from Email::Simple::Header, > + # we favor tabs for visibility (and space savings :P) > + if (length($_) >= $len && (/\n[^ \t]/s || !/\n/s)) { > + local $Text::Wrap::columns = $len; > + local $Text::Wrap::huge = 'overflow'; > + $pfx . wrap('', "\t", $v) . $self->{crlf}; > + } else { > + $pfx . $v . $self->{crlf}; > + } > + } @vals; > + ${$self->{hdr}} .= join('', @vals); > +} > + > sub mhdr_decode ($) { > eval { $MIME_Header->decode($_[0], Encode::FB_DEFAULT) } // $_[0]; > } > diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm > index 00f99ef7..dd55a2e6 100644 > --- a/lib/PublicInbox/IMAP.pm > +++ b/lib/PublicInbox/IMAP.pm > @@ -642,7 +642,7 @@ sub emit_rfc822_header { > $self->msg_more(${$eml->{hdr}}); > } > > -# n.b. this is sorted to be after any emit_eml_new ops > +# n.b. this is sorted to be after any op_eml_new ops > sub emit_rfc822_text { > my ($self, $k, undef, $bref) = @_; > $self->msg_more(" $k {".length($$bref)."}\r\n"); > @@ -668,9 +668,20 @@ sub to_crlf_full { > ${$_[0]} =~ s/\A[\r\n]*From [^\r\n]*\r\n//s; > } > > -sub op_crlf_bref { to_crlf_full($_[3]) } > +# used by PublicInbox::POP3, too > +sub op_crlf_bref { > + if ($_[0]->{ibx}->{appendheader}) { > + my $eml = PublicInbox::Eml->new($_[3]); > + $_[0]->{ibx}->append_headers($eml); > + ${$_[3]} = $eml->as_string; # replace bref > + } > + to_crlf_full($_[3]); > +} > > -sub op_crlf_hdr { to_crlf_full($_[4]->{hdr}) } > +sub op_crlf_hdr { # $_[4] = eml > + $_[0]->{ibx}->append_headers($_[4]) if $_[0]->{ibx}->{appendheader}; > + to_crlf_full($_[4]->{hdr}); > +} > > sub op_crlf_bdy { ${$_[4]->{bdy}} =~ s/(?<!\r)\n/\r\n/sg if $_[4]->{bdy} } > > diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm > index 9afbb478..d5108e4a 100644 > --- a/lib/PublicInbox/Inbox.pm > +++ b/lib/PublicInbox/Inbox.pm > @@ -410,4 +410,12 @@ sub mailboxid { # rfc 8474, 8620, 8621 > > sub thing_type { 'public inbox' } > > +sub append_headers { > + my ($self, $eml) = @_; > + # TODO: do we need MSGID expansions? > + for (@{$self->{appendheader}}) { > + $eml->header_append(split(/\s*:\s*/, $_, 2)); > + } > +} > + > 1; > diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm > index bf61bb0e..0b47cb8c 100644 > --- a/lib/PublicInbox/Mbox.pm > +++ b/lib/PublicInbox/Mbox.pm > @@ -89,15 +89,15 @@ sub emit_raw { > > sub msg_hdr ($$) { > my ($ctx, $eml) = @_; > - my $header_obj = $eml->header_obj; > > # drop potentially confusing headers, ssoma already should've dropped > # Lines and Content-Length > foreach my $d (qw(Lines Bytes Content-Length Status)) { > - $header_obj->header_set($d); > + $eml->header_set($d); > } > - my $crlf = $header_obj->crlf; > - my $buf = $header_obj->as_string; > + $ctx->{ibx}->append_headers($eml) if $ctx->{ibx}->{appendheader}; > + my $crlf = $eml->{crlf}; > + my $buf = ${$eml->{hdr}}; > # fixup old bug from import (pre-a0c07cba0e5d8b6a) > $buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s; > "From mboxrd\@z Thu Jan 1 00:00:00 1970" . $crlf . $buf . $crlf; > diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm > index 316b7775..91e6357f 100644 > --- a/lib/PublicInbox/NNTP.pm > +++ b/lib/PublicInbox/NNTP.pm > @@ -462,6 +462,8 @@ sub set_nntp_headers ($$) { > # *something* here is required for leafnode, try to follow > # RFC 5536 3.1.5... > $hdr->header_set('Path', $server_name . '!not-for-mail'); > + > + $ibx->append_headers($hdr) if $ibx->{appendheader}; > } > > sub art_lookup ($$$) { > diff --git a/lib/PublicInbox/POP3.pm b/lib/PublicInbox/POP3.pm > index d32793e4..863cb201 100644 > --- a/lib/PublicInbox/POP3.pm > +++ b/lib/PublicInbox/POP3.pm > @@ -229,7 +229,7 @@ sub retr_cb { # called by git->cat_async via ibx_async_cat > $self->close; > die "BUG: $hex != $oid"; > } > - PublicInbox::IMAP::to_crlf_full($bref); > + PublicInbox::IMAP::op_crlf_bref($self, undef, undef, $bref); > if (defined $top_nr) { > my ($hdr, $bdy) = split(/\r\n\r\n/, $$bref, 2); > $bref = \$hdr; > diff --git a/t/netd.t b/t/netd.t > index abdde124..47a0182f 100644 > --- a/t/netd.t > +++ b/t/netd.t > @@ -6,7 +6,8 @@ use Socket qw(IPPROTO_TCP SOL_SOCKET); > use PublicInbox::TestCommon; > # IO::Poll and Net::NNTP are part of the standard library, but > # distros may split them off... > -require_mods(qw(-imapd IO::Socket::SSL Mail::IMAPClient IO::Poll Net::NNTP)); > +require_mods(qw(-imapd IO::Socket::SSL Mail::IMAPClient IO::Poll Net::NNTP > + Net::POP3)); > my $imap_client = 'Mail::IMAPClient'; > $imap_client->can('starttls') or > plan skip_all => 'Mail::IMAPClient does not support TLS'; > @@ -21,6 +22,7 @@ unless (-r $key && -r $cert) { > use_ok 'PublicInbox::TLS'; > use_ok 'IO::Socket::SSL'; > require_git('2.6'); > +require_mods(qw(File::FcntlLock)) if $^O !~ /\A(?:linux|freebsd)\z/; > > my ($tmpdir, $for_destroy) = tmpdir(); > my $err = "$tmpdir/stderr.log"; > @@ -35,7 +37,8 @@ for (1..3) { > pipe(my ($r, $w)) or xbail "pipe: $!"; > push @pad_pipes, $r, $w; > }; > -my %srv = map { $_ => tcp_server() } qw(imap nntp imaps nntps); > +my %srv = map { $_ => tcp_server() } qw(imap nntp imaps nntps pop3 http); > +my ($hdr_key, $hdr_val) = qw(x-archive-source https://example.com/); > my $ibx = create_inbox 'netd', version => 2, > -primary_address => $addr, indexlevel => 'basic', sub { > my ($im, $ibx) = @_; > @@ -43,11 +46,14 @@ my $ibx = create_inbox 'netd', version => 2, > $pi_config = "$ibx->{inboxdir}/pi_config"; > open my $fh, '>', $pi_config or BAIL_OUT "open: $!"; > print $fh <<EOF or BAIL_OUT "print: $!"; > +[publicinbox] > + pop3state = $tmpdir/p3state > [publicinbox "netd"] > inboxdir = $ibx->{inboxdir} > address = $addr > indexlevel = basic > newsgroup = $group > + appendHeader = $hdr_key:$hdr_val > EOF > close $fh or BAIL_OUT "close: $!\n"; > }; > @@ -70,16 +76,45 @@ my %o = ( > SSL_verify_mode => SSL_VERIFY_PEER(), > SSL_ca_file => 'certs/test-ca.pem', > ); > + > +my $ok_inject = sub { > + my ($blob, $msg) = @_; > + my $eml = PublicInbox::Eml->new($blob); > + is_deeply([$eml->header($hdr_key)], [ $hdr_val ], "$msg header added"); > +}; > + > +{ > + my ($host, $port) = tcp_host_port($srv{imap}); > + my %mic_opt = (Server => $host, Port => $port, Uid => 1); > + $mic_opt{Authmechanism} = 'ANONYMOUS'; > + $mic_opt{Authcallback} = sub { '' }; > + my $mic = $imap_client->new(%mic_opt); > + ok($mic && $mic->examine("$group.0"), 'IMAP connected'); > + my $ret = $mic->fetch_hash(1, 'RFC822'); > + $ok_inject->($ret->{1}->{RFC822}, 'IMAP RFC822 (full)'); > + $ret = $mic->fetch_hash(1, 'RFC822.HEADER'); > + $ok_inject->($ret->{1}->{'RFC822.HEADER'}, 'IMAP RFC822.HEADER'); > +} > +{ > + my $nntp = Net::NNTP->new(my $host_port = tcp_host_port($srv{nntp})); > + ok($nntp && $nntp->group($group), 'NNTP group'); > + $ok_inject->(join('', @{$nntp->article(1)}), 'NNTP ->article'); > + $ok_inject->(join('', @{$nntp->head(1)}), 'NNTP ->head'); > +} > { > - my $c = tcp_connect($srv{imap}); > - my $msg = <$c>; > - like($msg, qr/IMAP4rev1/, 'connected to IMAP'); > + my ($host, $port) = tcp_host_port($srv{pop3}); > + my $pop3 = Net::POP3->new($host, Port => $port); > + my $locked_mb = ('e'x32)."\@$group"; > + ok($pop3 && $pop3->apop("$locked_mb.0", 'anonymous'), 'APOP connected'); > + $ok_inject->(join('', @{$pop3->get(1)}), 'POP3 ->get'); > } > { > - my $c = tcp_connect($srv{nntp}); > - my $msg = <$c>; > - like($msg, qr/^201 .*? ready - post via email/, 'connected to NNTP'); > + my $c = tcp_connect($srv{http}); > + ok($c and print $c <<EOM, 'HTTP connected'); > +GET /netd/20180720072141.GA15957\@example/raw HTTP/1.0\r\n\r > +EOM > + my $s = do { local $/; <$c> }; > + $ok_inject->((split(/\r\n\r\n/, $s, 2))[1], 'HTTP $MSGID/raw'); > } > > -# TODO: more tests > done_testing; > -- Pengutronix e.K. | | Steuerwalder Str. 21 | http://www.pengutronix.de/ | 31137 Hildesheim, Germany | Phone: +49-5121-206917-0 | Amtsgericht Hildesheim, HRA 2686 | Fax: +49-5121-206917-5555 |
