Konstantin Ryabitsev <[email protected]> wrote:
> On Wed, Jun 14, 2023 at 11:50:15PM +0000, Eric Wong wrote:
> > affects WWW and NNTP, but not IMAP/POP3. I'm not sure if I want
> > to reintroduce header injection in case there's some conflict
> > with DKIM or other signature mechanisms[1]
>
> I don't think we need to worry about it if we pick a header that's almost
> certain to not be included in the default DKIM signature set.
> X-Originally-Archived-At: or some other header is guaranteed to never be
> signed.
*shrug* I'm not sure how useful this is, actually.
-----------8<---------
Subject: [PATCH] support publicinbox.$FOO.appendHeader in read-only endpoints
This may be used to inject arbitrary headers in the raw message
from any read-only endpoints. This may be useful for mirrors to
indicate they're mirroring another source and can't edit/remove
messages easily.
This is set per-inbox and supports multiple key:value pairs:
[publicinbox "foo"]
appendheader = KEY1:VALUE1
appendheader = KEY2:VALUE2
No variable expansion is currently supported as it's unclear if
it's necessary (e.g. for Message-ID in URLs).
Link: https://public-inbox.org/meta/20230615-focal-erosion-poop-df8246@meerkat/
---
lib/PublicInbox/Config.pm | 2 +-
lib/PublicInbox/Eml.pm | 20 +++++++++++++++
lib/PublicInbox/IMAP.pm | 17 ++++++++++---
lib/PublicInbox/Inbox.pm | 8 ++++++
lib/PublicInbox/Mbox.pm | 8 +++---
lib/PublicInbox/NNTP.pm | 2 ++
lib/PublicInbox/POP3.pm | 2 +-
t/netd.t | 53 ++++++++++++++++++++++++++++++++-------
8 files changed, 94 insertions(+), 18 deletions(-)
diff --git a/lib/PublicInbox/Config.pm b/lib/PublicInbox/Config.pm
index 2f1b4122..c3ec1793 100644
--- a/lib/PublicInbox/Config.pm
+++ b/lib/PublicInbox/Config.pm
@@ -457,7 +457,7 @@ sub _fill_ibx {
# more things to encourage decentralization
for my $k (qw(address altid nntpmirror imapmirror
coderepo hide listid url
- infourl watchheader
+ infourl watchheader appendheader
nntpserver imapserver pop3server)) {
my $v = $self->{"$pfx.$k"} // next;
$ibx->{$k} = _array($v);
diff --git a/lib/PublicInbox/Eml.pm b/lib/PublicInbox/Eml.pm
index 8b999e1a..3f7e27e0 100644
--- a/lib/PublicInbox/Eml.pm
+++ b/lib/PublicInbox/Eml.pm
@@ -392,6 +392,26 @@ sub header_str_set {
header_set($self, $name, @vals);
}
+# only used for publicinbox.$NAME.appendHeader
+sub header_append {
+ my ($self, $pfx, @vals) = @_;
+ $pfx .= ': ';
+ my $len = 78 - length($pfx);
+ @vals = map {;
+ utf8::encode(my $v = $_); # to bytes, support SMTPUTF8
+ # folding differs from Email::Simple::Header,
+ # we favor tabs for visibility (and space savings :P)
+ if (length($_) >= $len && (/\n[^ \t]/s || !/\n/s)) {
+ local $Text::Wrap::columns = $len;
+ local $Text::Wrap::huge = 'overflow';
+ $pfx . wrap('', "\t", $v) . $self->{crlf};
+ } else {
+ $pfx . $v . $self->{crlf};
+ }
+ } @vals;
+ ${$self->{hdr}} .= join('', @vals);
+}
+
sub mhdr_decode ($) {
eval { $MIME_Header->decode($_[0], Encode::FB_DEFAULT) } // $_[0];
}
diff --git a/lib/PublicInbox/IMAP.pm b/lib/PublicInbox/IMAP.pm
index 00f99ef7..dd55a2e6 100644
--- a/lib/PublicInbox/IMAP.pm
+++ b/lib/PublicInbox/IMAP.pm
@@ -642,7 +642,7 @@ sub emit_rfc822_header {
$self->msg_more(${$eml->{hdr}});
}
-# n.b. this is sorted to be after any emit_eml_new ops
+# n.b. this is sorted to be after any op_eml_new ops
sub emit_rfc822_text {
my ($self, $k, undef, $bref) = @_;
$self->msg_more(" $k {".length($$bref)."}\r\n");
@@ -668,9 +668,20 @@ sub to_crlf_full {
${$_[0]} =~ s/\A[\r\n]*From [^\r\n]*\r\n//s;
}
-sub op_crlf_bref { to_crlf_full($_[3]) }
+# used by PublicInbox::POP3, too
+sub op_crlf_bref {
+ if ($_[0]->{ibx}->{appendheader}) {
+ my $eml = PublicInbox::Eml->new($_[3]);
+ $_[0]->{ibx}->append_headers($eml);
+ ${$_[3]} = $eml->as_string; # replace bref
+ }
+ to_crlf_full($_[3]);
+}
-sub op_crlf_hdr { to_crlf_full($_[4]->{hdr}) }
+sub op_crlf_hdr { # $_[4] = eml
+ $_[0]->{ibx}->append_headers($_[4]) if $_[0]->{ibx}->{appendheader};
+ to_crlf_full($_[4]->{hdr});
+}
sub op_crlf_bdy { ${$_[4]->{bdy}} =~ s/(?<!\r)\n/\r\n/sg if $_[4]->{bdy} }
diff --git a/lib/PublicInbox/Inbox.pm b/lib/PublicInbox/Inbox.pm
index 9afbb478..d5108e4a 100644
--- a/lib/PublicInbox/Inbox.pm
+++ b/lib/PublicInbox/Inbox.pm
@@ -410,4 +410,12 @@ sub mailboxid { # rfc 8474, 8620, 8621
sub thing_type { 'public inbox' }
+sub append_headers {
+ my ($self, $eml) = @_;
+ # TODO: do we need MSGID expansions?
+ for (@{$self->{appendheader}}) {
+ $eml->header_append(split(/\s*:\s*/, $_, 2));
+ }
+}
+
1;
diff --git a/lib/PublicInbox/Mbox.pm b/lib/PublicInbox/Mbox.pm
index bf61bb0e..0b47cb8c 100644
--- a/lib/PublicInbox/Mbox.pm
+++ b/lib/PublicInbox/Mbox.pm
@@ -89,15 +89,15 @@ sub emit_raw {
sub msg_hdr ($$) {
my ($ctx, $eml) = @_;
- my $header_obj = $eml->header_obj;
# drop potentially confusing headers, ssoma already should've dropped
# Lines and Content-Length
foreach my $d (qw(Lines Bytes Content-Length Status)) {
- $header_obj->header_set($d);
+ $eml->header_set($d);
}
- my $crlf = $header_obj->crlf;
- my $buf = $header_obj->as_string;
+ $ctx->{ibx}->append_headers($eml) if $ctx->{ibx}->{appendheader};
+ my $crlf = $eml->{crlf};
+ my $buf = ${$eml->{hdr}};
# fixup old bug from import (pre-a0c07cba0e5d8b6a)
$buf =~ s/\A[\r\n]*From [^\r\n]*\r?\n//s;
"From mboxrd\@z Thu Jan 1 00:00:00 1970" . $crlf . $buf . $crlf;
diff --git a/lib/PublicInbox/NNTP.pm b/lib/PublicInbox/NNTP.pm
index 316b7775..91e6357f 100644
--- a/lib/PublicInbox/NNTP.pm
+++ b/lib/PublicInbox/NNTP.pm
@@ -462,6 +462,8 @@ sub set_nntp_headers ($$) {
# *something* here is required for leafnode, try to follow
# RFC 5536 3.1.5...
$hdr->header_set('Path', $server_name . '!not-for-mail');
+
+ $ibx->append_headers($hdr) if $ibx->{appendheader};
}
sub art_lookup ($$$) {
diff --git a/lib/PublicInbox/POP3.pm b/lib/PublicInbox/POP3.pm
index d32793e4..863cb201 100644
--- a/lib/PublicInbox/POP3.pm
+++ b/lib/PublicInbox/POP3.pm
@@ -229,7 +229,7 @@ sub retr_cb { # called by git->cat_async via ibx_async_cat
$self->close;
die "BUG: $hex != $oid";
}
- PublicInbox::IMAP::to_crlf_full($bref);
+ PublicInbox::IMAP::op_crlf_bref($self, undef, undef, $bref);
if (defined $top_nr) {
my ($hdr, $bdy) = split(/\r\n\r\n/, $$bref, 2);
$bref = \$hdr;
diff --git a/t/netd.t b/t/netd.t
index abdde124..47a0182f 100644
--- a/t/netd.t
+++ b/t/netd.t
@@ -6,7 +6,8 @@ use Socket qw(IPPROTO_TCP SOL_SOCKET);
use PublicInbox::TestCommon;
# IO::Poll and Net::NNTP are part of the standard library, but
# distros may split them off...
-require_mods(qw(-imapd IO::Socket::SSL Mail::IMAPClient IO::Poll Net::NNTP));
+require_mods(qw(-imapd IO::Socket::SSL Mail::IMAPClient IO::Poll Net::NNTP
+ Net::POP3));
my $imap_client = 'Mail::IMAPClient';
$imap_client->can('starttls') or
plan skip_all => 'Mail::IMAPClient does not support TLS';
@@ -21,6 +22,7 @@ unless (-r $key && -r $cert) {
use_ok 'PublicInbox::TLS';
use_ok 'IO::Socket::SSL';
require_git('2.6');
+require_mods(qw(File::FcntlLock)) if $^O !~ /\A(?:linux|freebsd)\z/;
my ($tmpdir, $for_destroy) = tmpdir();
my $err = "$tmpdir/stderr.log";
@@ -35,7 +37,8 @@ for (1..3) {
pipe(my ($r, $w)) or xbail "pipe: $!";
push @pad_pipes, $r, $w;
};
-my %srv = map { $_ => tcp_server() } qw(imap nntp imaps nntps);
+my %srv = map { $_ => tcp_server() } qw(imap nntp imaps nntps pop3 http);
+my ($hdr_key, $hdr_val) = qw(x-archive-source https://example.com/);
my $ibx = create_inbox 'netd', version => 2,
-primary_address => $addr, indexlevel => 'basic', sub {
my ($im, $ibx) = @_;
@@ -43,11 +46,14 @@ my $ibx = create_inbox 'netd', version => 2,
$pi_config = "$ibx->{inboxdir}/pi_config";
open my $fh, '>', $pi_config or BAIL_OUT "open: $!";
print $fh <<EOF or BAIL_OUT "print: $!";
+[publicinbox]
+ pop3state = $tmpdir/p3state
[publicinbox "netd"]
inboxdir = $ibx->{inboxdir}
address = $addr
indexlevel = basic
newsgroup = $group
+ appendHeader = $hdr_key:$hdr_val
EOF
close $fh or BAIL_OUT "close: $!\n";
};
@@ -70,16 +76,45 @@ my %o = (
SSL_verify_mode => SSL_VERIFY_PEER(),
SSL_ca_file => 'certs/test-ca.pem',
);
+
+my $ok_inject = sub {
+ my ($blob, $msg) = @_;
+ my $eml = PublicInbox::Eml->new($blob);
+ is_deeply([$eml->header($hdr_key)], [ $hdr_val ], "$msg header added");
+};
+
+{
+ my ($host, $port) = tcp_host_port($srv{imap});
+ my %mic_opt = (Server => $host, Port => $port, Uid => 1);
+ $mic_opt{Authmechanism} = 'ANONYMOUS';
+ $mic_opt{Authcallback} = sub { '' };
+ my $mic = $imap_client->new(%mic_opt);
+ ok($mic && $mic->examine("$group.0"), 'IMAP connected');
+ my $ret = $mic->fetch_hash(1, 'RFC822');
+ $ok_inject->($ret->{1}->{RFC822}, 'IMAP RFC822 (full)');
+ $ret = $mic->fetch_hash(1, 'RFC822.HEADER');
+ $ok_inject->($ret->{1}->{'RFC822.HEADER'}, 'IMAP RFC822.HEADER');
+}
+{
+ my $nntp = Net::NNTP->new(my $host_port = tcp_host_port($srv{nntp}));
+ ok($nntp && $nntp->group($group), 'NNTP group');
+ $ok_inject->(join('', @{$nntp->article(1)}), 'NNTP ->article');
+ $ok_inject->(join('', @{$nntp->head(1)}), 'NNTP ->head');
+}
{
- my $c = tcp_connect($srv{imap});
- my $msg = <$c>;
- like($msg, qr/IMAP4rev1/, 'connected to IMAP');
+ my ($host, $port) = tcp_host_port($srv{pop3});
+ my $pop3 = Net::POP3->new($host, Port => $port);
+ my $locked_mb = ('e'x32)."\@$group";
+ ok($pop3 && $pop3->apop("$locked_mb.0", 'anonymous'), 'APOP connected');
+ $ok_inject->(join('', @{$pop3->get(1)}), 'POP3 ->get');
}
{
- my $c = tcp_connect($srv{nntp});
- my $msg = <$c>;
- like($msg, qr/^201 .*? ready - post via email/, 'connected to NNTP');
+ my $c = tcp_connect($srv{http});
+ ok($c and print $c <<EOM, 'HTTP connected');
+GET /netd/20180720072141.GA15957\@example/raw HTTP/1.0\r\n\r
+EOM
+ my $s = do { local $/; <$c> };
+ $ok_inject->((split(/\r\n\r\n/, $s, 2))[1], 'HTTP $MSGID/raw');
}
-# TODO: more tests
done_testing;
