It is sometimes useful to test Patchwork on real data. However, storing emails of real people in a way that is recognized by Patchwork may cause problems (ex. accidental spam). Add an option to ingress that anonymizes headers of imported emails. Note that the contents of the email itself are not anonymized, ex. Signed-Off-By commit headers are left intact. Anonymization also removes almost all headers, except for a small, explicitly specified subset.
Signed-off-by: Franciszek Stachura <[email protected]> --- cmd/pw/ingress/main.go | 25 +++++++------ pkg/mail/parser.go | 83 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 89 insertions(+), 19 deletions(-) diff --git a/cmd/pw/ingress/main.go b/cmd/pw/ingress/main.go index f951b0a..274e766 100644 --- a/cmd/pw/ingress/main.go +++ b/cmd/pw/ingress/main.go @@ -29,9 +29,10 @@ import ( ) type CLI struct { - Stdin bool `short:"i" help:"Read one email from stdin and exit."` - Mbox bool `short:"m" help:"Read all emails in mbox format from stdin."` - ListID string `short:"l" help:"Force List-ID value instead of reading it from email headers."` + Stdin bool `short:"i" help:"Read one email from stdin and exit."` + Mbox bool `short:"m" help:"Read all emails in mbox format from stdin."` + ListID string `short:"l" help:"Force List-ID value instead of reading it from email headers."` + AnonymizeToList *string `help:"ID of list to add messages to after partial anonymization."` } func (c *CLI) Run(ctx context.Context) error { @@ -52,6 +53,7 @@ func (c *CLI) Run(ctx context.Context) error { if c.Stdin || c.Mbox { var dupErr *mail.DuplicateMailError + var parseErr *mail.ParseError var err error if c.Mbox { @@ -62,17 +64,17 @@ func (c *CLI) Run(ctx context.Context) error { if err != nil { break } - err = mail.ParseMail(ctx, database, msg, c.ListID) - if errors.As(err, &dupErr) { + err = mail.ParseMail(ctx, database, msg, c.AnonymizeToList, c.ListID) + if errors.As(err, &dupErr) || errors.As(err, &parseErr) { log.Debugf("ignoring %s", err) } else if err != nil { break } } } else { - err = mail.ParseMail(ctx, database, os.Stdin, c.ListID) + err = mail.ParseMail(ctx, database, os.Stdin, c.AnonymizeToList, c.ListID) } - if errors.As(err, &dupErr) { + if errors.As(err, &dupErr) || errors.As(err, &parseErr) { log.Debugf("ignoring %s", err) } else if err != nil && !errors.Is(err, io.EOF) { return fmt.Errorf("smtp: %w", err) @@ -106,7 +108,7 @@ func (c *CLI) Run(ctx context.Context) error { func (c *CLI) startSMTPServer(ctx context.Context) (net.Listener, *smtp.Server, error) { cfg := pw.GetConfig(ctx) - s := smtp.NewServer(&backend{ctx: ctx, listID: c.ListID}) + s := smtp.NewServer(&backend{ctx: ctx, listID: c.ListID, anonymizeListID: c.AnonymizeToList}) s.Addr = cfg.Ingress.Listen s.Domain = "localhost" s.ReadTimeout = 30 * time.Second @@ -129,8 +131,9 @@ func (c *CLI) startSMTPServer(ctx context.Context) (net.Listener, *smtp.Server, } type backend struct { - ctx context.Context - listID string + ctx context.Context + listID string + anonymizeListID *string } func (b *backend) NewSession(c *smtp.Conn) (smtp.Session, error) { @@ -200,7 +203,7 @@ func (s *session) Data(r io.Reader) error { err = mail.ParseMail( s.backend.ctx, pw.GetDB(s.backend.ctx), - bytes.NewReader(data), s.backend.listID, + bytes.NewReader(data), s.backend.anonymizeListID, s.backend.listID, ) if err != nil { var dupErr *mail.DuplicateMailError diff --git a/pkg/mail/parser.go b/pkg/mail/parser.go index 6f405cd..328b927 100644 --- a/pkg/mail/parser.go +++ b/pkg/mail/parser.go @@ -7,6 +7,8 @@ package mail import ( "context" + "crypto/md5" + "encoding/base64" "fmt" "io" "strings" @@ -48,7 +50,64 @@ type parser struct { refs []string } -func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...string) error { +func anonymizeHeaders(toList string, header mail.Header, from *mail.Address, msgid string) (mail.Header, *mail.Address, string) { + newHeader := mail.Header{} + + toListSplit := strings.Split(toList, "@") + toListDomain := "example.com" + if len(toListSplit) > 1 { + toListDomain = toListSplit[len(toListSplit)-1] + } + + replyToFields := header.FieldsByKey("In-Reply-To") + for { + hasMore := replyToFields.Next() + if !hasMore { + break + } + + text, err := replyToFields.Text() + if err != nil { + break + } + fieldEncodedMd5 := md5.Sum([]byte(text)) + fieldEncoded := base64.URLEncoding.EncodeToString(fieldEncodedMd5[:]) + fieldEncoded = fmt.Sprintf("<%s@%s>", fieldEncoded, toListDomain) + newHeader.Add("In-Reply-To", fieldEncoded) + } + + referencesFields := header.FieldsByKey("References") + for { + hasMore := replyToFields.Next() + if !hasMore { + break + } + + text, err := referencesFields.Text() + if err != nil { + break + } + fieldEncodedMd5 := md5.Sum([]byte(text)) + fieldEncoded := base64.URLEncoding.EncodeToString(fieldEncodedMd5[:]) + fieldEncoded = fmt.Sprintf("<%s@%s>", fieldEncoded, toListDomain) + newHeader.Add("References", fieldEncoded) + } + + emailEncoded := md5.Sum([]byte(from.Address)) + fromEmail := fmt.Sprintf("%s@%s", base64.URLEncoding.EncodeToString(emailEncoded[:]), toListDomain) + from = &mail.Address{Name: from.Name, Address: fromEmail} + + msgidEncoded := md5.Sum([]byte(msgid)) + msgid = fmt.Sprintf("%s@%s", base64.URLEncoding.EncodeToString(msgidEncoded[:]), toListDomain) + + newHeader.Set("List-ID", fmt.Sprintf("<%s>", toList)) + newHeader.SetMessageID(msgid) + newHeader.SetAddressList("From", []*mail.Address{from}) + + return newHeader, from, msgid +} + +func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, anonymizeToList *string, listid ...string) error { m, err := mail.CreateReader(r) if err != nil { return ParseErr("read message: %v", err) @@ -56,30 +115,38 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str // basic sanity checks + header := m.Header + if strings.EqualFold(m.Header.Get("X-Patchwork-Hint"), "ignore") { log.Debugf("ignoring email due to hint") return nil } - subject, err := m.Header.Subject() + subject, err := header.Subject() if err != nil { return ParseErr("subject: %v", err) } - date, err := m.Header.Date() + date, err := header.Date() if err != nil { log.Warnf("date: %v", err) } if date.IsZero() { date = time.Now() } - msgid, err := m.Header.MessageID() + msgid, err := header.MessageID() if err != nil { return ParseErr("message-id: %v", err) } - from, err := mail.ParseAddress(m.Header.Get("From")) + from, err := mail.ParseAddress(header.Get("From")) if err != nil { return ParseErr("from: %v", err) } + if anonymizeToList != nil { + header, from, msgid = anonymizeHeaders(*anonymizeToList, header, from, msgid) + header.SetDate(date) + header.SetSubject(subject) + } + queries, err := db.Begin(ctx, database) if err != nil { return err @@ -88,7 +155,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str p := parser{ db: queries, - header: &m.Header, + header: &header, subject: subject, date: date, msgid: "<" + msgid + ">", @@ -115,7 +182,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str p.parseSeriesMarker(isComment) p.version = ParseVersion(p.subject, p.prefixes) - p.refs = FindReferences(&m.Header) + p.refs = FindReferences(&header) log.Debugf("series marker: n=%d total=%d version=%d comment=%v refs=%v", p.number, p.total, p.version, isComment, p.refs) @@ -131,7 +198,7 @@ func ParseMail(ctx context.Context, database *bun.DB, r io.Reader, listid ...str log.Debugf("no diff or comment content, skipping") return nil } - p.content.headers = FormatHeaders(&m.Header) + p.content.headers = FormatHeaders(&header) p.pullURL = ParsePullRequest(p.content.comment) switch { -- 2.55.0 _______________________________________________ Patchwork mailing list [email protected] https://lists.ozlabs.org/listinfo/patchwork
