This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 288cd43bdc GH-48623: [CI][Archery][Dev] Add missing headers to email
reports (#48624)
288cd43bdc is described below
commit 288cd43bdc2a8a38df414832f4aa985c203a0ffc
Author: Sutou Kouhei <[email protected]>
AuthorDate: Tue Jan 13 12:08:03 2026 +0900
GH-48623: [CI][Archery][Dev] Add missing headers to email reports (#48624)
### Rationale for this change
Our email reports miss the following headers:
* `MIME-Version: 1.0`
* `Content-Type: text/plain; charset="utf-8"`
* `Message-Id: ${AUTO_GENERATED_MESSAGE_ID}`
* `Date: ${DATE_IN_RFC_2822}`
### What changes are included in this PR?
Add these headers.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* GitHub Issue: #48623
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.github/workflows/cpp_extra.yml | 2 +
.github/workflows/package_linux.yml | 2 +
.github/workflows/r_extra.yml | 38 ++++++++-------
dev/archery/archery/ci/cli.py | 28 ++++++++---
dev/archery/archery/crossbow/cli.py | 54 ++++++++++++++-------
dev/archery/archery/crossbow/reports.py | 43 ++++++++++++++---
.../{email-report.txt => nightly-email-report.txt} | 5 ++
.../fixtures/token-expiration-email-report.txt | 14 ++++++
dev/archery/archery/crossbow/tests/test_reports.py | 55 ++++++++++++++++++----
.../archery/templates/email_nightly_report.txt.j2 | 10 +---
.../templates/email_token_expiration.txt.j2 | 11 ++---
.../archery/templates/email_workflow_report.txt.j2 | 10 +---
12 files changed, 193 insertions(+), 79 deletions(-)
diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml
index 614649660d..f353fe8634 100644
--- a/.github/workflows/cpp_extra.yml
+++ b/.github/workflows/cpp_extra.yml
@@ -39,6 +39,7 @@ on:
- 'ci/scripts/util_*'
- 'cpp/**'
- 'compose.yaml'
+ - 'dev/archery/archery/**'
- 'format/Flight.proto'
- 'testing'
tags:
@@ -61,6 +62,7 @@ on:
- 'ci/scripts/util_*'
- 'cpp/**'
- 'compose.yaml'
+ - 'dev/archery/archery/**'
- 'format/Flight.proto'
- 'testing'
types:
diff --git a/.github/workflows/package_linux.yml
b/.github/workflows/package_linux.yml
index 3e4b759215..1d2ae61f1e 100644
--- a/.github/workflows/package_linux.yml
+++ b/.github/workflows/package_linux.yml
@@ -29,6 +29,7 @@ on:
- '.github/workflows/report_ci.yml'
- 'cpp/**'
- 'c_glib/**'
+ - 'dev/archery/archery/**'
- 'dev/release/binary-task.rb'
- 'dev/release/verify-apt.sh'
- 'dev/release/verify-yum.sh'
@@ -43,6 +44,7 @@ on:
- '.github/workflows/report_ci.yml'
- 'cpp/**'
- 'c_glib/**'
+ - 'dev/archery/archery/**'
- 'dev/release/binary-task.rb'
- 'dev/release/verify-apt.sh'
- 'dev/release/verify-yum.sh'
diff --git a/.github/workflows/r_extra.yml b/.github/workflows/r_extra.yml
index 5b11aa12ee..7482725059 100644
--- a/.github/workflows/r_extra.yml
+++ b/.github/workflows/r_extra.yml
@@ -27,15 +27,16 @@ on:
- '.github/workflows/check_labels.yml'
- '.github/workflows/r_extra.yml'
- '.github/workflows/report_ci.yml'
- - "ci/docker/**"
- - "ci/etc/rprofile"
- - "ci/scripts/PKGBUILD"
- - "ci/scripts/cpp_*.sh"
- - "ci/scripts/install_minio.sh"
- - "ci/scripts/r_*.sh"
- - "cpp/**"
- - "compose.yaml"
- - "r/**"
+ - 'ci/docker/**'
+ - 'ci/etc/rprofile'
+ - 'ci/scripts/PKGBUILD'
+ - 'ci/scripts/cpp_*.sh'
+ - 'ci/scripts/install_minio.sh'
+ - 'ci/scripts/r_*.sh'
+ - 'cpp/**'
+ - 'compose.yaml'
+ - 'dev/archery/archery/**'
+ - 'r/**'
tags:
- '**'
pull_request:
@@ -44,15 +45,16 @@ on:
- '.github/workflows/check_labels.yml'
- '.github/workflows/r_extra.yml'
- '.github/workflows/report_ci.yml'
- - "ci/docker/**"
- - "ci/etc/rprofile"
- - "ci/scripts/PKGBUILD"
- - "ci/scripts/cpp_*.sh"
- - "ci/scripts/install_minio.sh"
- - "ci/scripts/r_*.sh"
- - "cpp/**"
- - "compose.yaml"
- - "r/**"
+ - 'ci/docker/**'
+ - 'ci/etc/rprofile'
+ - 'ci/scripts/PKGBUILD'
+ - 'ci/scripts/cpp_*.sh'
+ - 'ci/scripts/install_minio.sh'
+ - 'ci/scripts/r_*.sh'
+ - 'cpp/**'
+ - 'compose.yaml'
+ - 'dev/archery/archery/**'
+ - 'r/**'
types:
- labeled
- opened
diff --git a/dev/archery/archery/ci/cli.py b/dev/archery/archery/ci/cli.py
index bf7b68d532..5597dff733 100644
--- a/dev/archery/archery/ci/cli.py
+++ b/dev/archery/archery/ci/cli.py
@@ -73,6 +73,22 @@ def report_chat(obj, workflow_id, send, repository, ignore,
webhook,
output.write(report_chat.render("workflow_report"))
+class WorkflowEmailReport(EmailReport):
+ def __init__(self, **kwargs):
+ super().__init__('workflow_report', **kwargs)
+
+ def date(self):
+ return self.report.datetime
+
+ def subject(self):
+ workflow = self.report
+ date = self.date().strftime('%Y-%m-%d')
+ return (
+ f'[{date}] Arrow Build Report for Job {workflow.name}: '
+ f'{len(workflow.failed_jobs())} failed'
+ )
+
+
@ci.command()
@click.argument('workflow_id', required=True)
@click.option('--sender-name', '-n',
@@ -105,9 +121,10 @@ def report_email(obj, workflow_id, sender_name,
sender_email, recipient_email,
"""
output = obj['output']
- email_report = EmailReport(
- report=Workflow(workflow_id, repository,
- ignore_job=ignore, gh_token=obj['github_token']),
+ workflow = Workflow(workflow_id, repository,
+ ignore_job=ignore, gh_token=obj['github_token'])
+ email_report = WorkflowEmailReport(
+ report=workflow,
sender_name=sender_name,
sender_email=sender_email,
recipient_email=recipient_email
@@ -119,8 +136,7 @@ def report_email(obj, workflow_id, sender_name,
sender_email, recipient_email,
smtp_password=smtp_password,
smtp_server=smtp_server,
smtp_port=smtp_port,
- recipient_email=recipient_email,
- message=email_report.render("workflow_report")
+ report=email_report
)
else:
- output.write(email_report.render("workflow_report"))
+ output.write(str(email_report.render()))
diff --git a/dev/archery/archery/crossbow/cli.py
b/dev/archery/archery/crossbow/cli.py
index c73c4d1ff7..10aa3dedf4 100644
--- a/dev/archery/archery/crossbow/cli.py
+++ b/dev/archery/archery/crossbow/cli.py
@@ -343,6 +343,22 @@ def latest_prefix(obj, prefix, fetch):
click.echo(latest.branch)
+class NightlyEmailReport(EmailReport):
+ def __init__(self, **kwargs):
+ super().__init__('nightly_report', **kwargs)
+
+ def subject(self):
+ report = self.report
+ n_errors = len(report.tasks_by_state['error'])
+ n_failures = len(report.tasks_by_state['failure'])
+ n_pendings = len(report.tasks_by_state['pending'])
+ return (
+ f'[NIGHTLY] Arrow Build Report for Job {report.job.branch}: '
+ f'{n_errors + n_failures} failed, '
+ f'{n_pendings} pending'
+ )
+
+
@crossbow.command()
@click.argument('job-name', required=True)
@click.option('--sender-name', '-n',
@@ -382,8 +398,9 @@ def report(obj, job_name, sender_name, sender_email,
recipient_email,
queue.fetch()
job = queue.get(job_name)
- email_report = EmailReport(
- report=Report(job),
+ report = Report(job)
+ email_report = NightlyEmailReport(
+ report=report,
sender_name=sender_name,
sender_email=sender_email,
recipient_email=recipient_email
@@ -401,11 +418,10 @@ def report(obj, job_name, sender_name, sender_email,
recipient_email,
smtp_password=smtp_password,
smtp_server=smtp_server,
smtp_port=smtp_port,
- recipient_email=recipient_email,
- message=email_report.render("nightly_report")
+ report=email_report
)
else:
- output.write(email_report.render("nightly_report"))
+ output.write(str(email_report.render()))
@crossbow.command()
@@ -601,6 +617,17 @@ def delete_old_branches(obj, dry_run, days, maximum):
print(batch)
+class TokenExpirationEmailReport(EmailReport):
+ def __init__(self, **kwargs):
+ super().__init__('token_expiration', **kwargs)
+
+ def subject(self):
+ token_expiration_date = self.report.token_expiration_date
+ return (
+ f'[CI] Arrow Crossbow Token Expiration in {token_expiration_date}'
+ )
+
+
@crossbow.command()
@click.option('--days', default=30,
help='Notification will be sent if expiration date is '
@@ -645,23 +672,18 @@ def notify_token_expiration(obj, days, sender_name,
sender_email,
self.token_expiration_date = token_expiration_date
self.days_left = days_left
- email_report = EmailReport(
- report=TokenExpirationReport(
- token_expiration_date or "ALREADY_EXPIRED", days_left),
- sender_name=sender_name,
- sender_email=sender_email,
- recipient_email=recipient_email
- )
+ if not token_expiration_date:
+ token_expiration_date = 'ALREADY_EXPIRED'
+ report = TokenExpirationReport(token_expiration_date, days_left)
+ email_report = TokenExpirationEmailReport(report)
- message = email_report.render("token_expiration").strip()
if send:
ReportUtils.send_email(
smtp_user=smtp_user,
smtp_password=smtp_password,
smtp_server=smtp_server,
smtp_port=smtp_port,
- recipient_email=recipient_email,
- message=message
+ report=email_report
)
else:
- output.write(message)
+ output.write(str(email_report.render()))
diff --git a/dev/archery/archery/crossbow/reports.py
b/dev/archery/archery/crossbow/reports.py
index 32962410d6..a2c0487a2b 100644
--- a/dev/archery/archery/crossbow/reports.py
+++ b/dev/archery/archery/crossbow/reports.py
@@ -17,6 +17,10 @@
import collections
import csv
+import datetime
+import email.headerregistry
+import email.message
+import email.utils
import operator
import fnmatch
import functools
@@ -246,7 +250,7 @@ class ReportUtils:
@classmethod
def send_email(cls, smtp_user, smtp_password, smtp_server, smtp_port,
- recipient_email, message):
+ report):
from smtplib import SMTP, SMTP_SSL
if smtp_port == 465:
@@ -259,7 +263,8 @@ class ReportUtils:
else:
smtp.starttls()
smtp.login(smtp_user, smtp_password)
- smtp.sendmail(smtp_user, recipient_email, message)
+ message = report.render()
+ smtp.send_message(smtp_user, report.recipient_email, message)
@classmethod
def write_csv(cls, report, add_headers=True):
@@ -271,11 +276,6 @@ class ReportUtils:
class EmailReport(JinjaReport):
- templates = {
- 'nightly_report': 'email_nightly_report.txt.j2',
- 'token_expiration': 'email_token_expiration.txt.j2',
- 'workflow_report': 'email_workflow_report.txt.j2',
- }
fields = [
'report',
'sender_name',
@@ -283,6 +283,35 @@ class EmailReport(JinjaReport):
'recipient_email',
]
+ def __init__(self, template_name, **kwargs):
+ self._template_name = template_name
+ super().__init__(**kwargs)
+
+ @property
+ def templates(self):
+ return {
+ self._template_name: f'email_{self._template_name}.txt.j2',
+ }
+
+ def date(self):
+ return None
+
+ def render(self):
+ message = email.message.EmailMessage()
+ message.set_charset('utf-8')
+ message['Message-Id'] = email.utils.make_msgid()
+ date = self.date()
+ if isinstance(date, datetime.datetime):
+ message['Date'] = date
+ else:
+ message['Date'] = email.utils.formatdate(date)
+ message['From'] = email.headerregistry.Address(
+ self.sender_name, addr_spec=self.sender_email)
+ message['To'] =
email.headerregistry.Address(addr_spec=self.recipient_email)
+ message['Subject'] = self.subject()
+ message.set_content(super().render(self._template_name))
+ return message
+
class CommentReport(Report):
diff --git a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt
b/dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt
similarity index 83%
rename from dev/archery/archery/crossbow/tests/fixtures/email-report.txt
rename to dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt
index c29cafd393..5e7b8e9c67 100644
--- a/dev/archery/archery/crossbow/tests/fixtures/email-report.txt
+++ b/dev/archery/archery/crossbow/tests/fixtures/nightly-email-report.txt
@@ -1,6 +1,11 @@
+MIME-Version: 1.0
+Message-Id: <message-id>
+Date: date
From: Sender Reporter <[email protected]>
To: [email protected]
Subject: [NIGHTLY] Arrow Build Report for Job ursabot-1: 2 failed, 1 pending
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
Arrow Build Report for Job ursabot-1
diff --git
a/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt
b/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt
new file mode 100644
index 0000000000..1f8ccbf30c
--- /dev/null
+++
b/dev/archery/archery/crossbow/tests/fixtures/token-expiration-email-report.txt
@@ -0,0 +1,14 @@
+MIME-Version: 1.0
+Message-Id: <message-id>
+Date: date
+From: Sender Reporter <[email protected]>
+To: [email protected]
+Subject: [CI] Arrow Crossbow Token Expiration in 2026-01-17
+Content-Type: text/plain; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+The Arrow Crossbow Token will expire in 7 days.
+
+Please generate a new Token. Send it to Apache INFRA to update the
+CROSSBOW_GITHUB_TOKEN. Update it on the crossbow repository and in
+the Azure pipelines.
diff --git a/dev/archery/archery/crossbow/tests/test_reports.py
b/dev/archery/archery/crossbow/tests/test_reports.py
index 620b4c78bb..02012d2f1b 100644
--- a/dev/archery/archery/crossbow/tests/test_reports.py
+++ b/dev/archery/archery/crossbow/tests/test_reports.py
@@ -15,11 +15,12 @@
# specific language governing permissions and limitations
# under the License.
+import re
import textwrap
+from archery.crossbow.cli import (NightlyEmailReport,
TokenExpirationEmailReport)
from archery.crossbow.core import yaml
-from archery.crossbow.reports import (ChatReport, CommentReport, EmailReport,
- Report)
+from archery.crossbow.reports import (ChatReport, CommentReport, Report)
def test_crossbow_comment_formatter(load_fixture):
@@ -71,19 +72,55 @@ def
test_crossbow_chat_report_extra_message_success(load_fixture):
assert report_chat.render("text") == textwrap.dedent(expected_msg)
-def test_crossbow_email_report(load_fixture):
- expected_msg = load_fixture('email-report.txt')
+def test_crossbow_nightly_email_report(load_fixture):
+ expected_msg = load_fixture('nightly-email-report.txt')
job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
report = Report(job)
assert report.tasks_by_state is not None
- email_report = EmailReport(report=report, sender_name="Sender Reporter",
- sender_email="[email protected]",
- recipient_email="[email protected]")
+ email_report = NightlyEmailReport(
+ report=report,
+ sender_name='Sender Reporter',
+ sender_email='[email protected]',
+ recipient_email='[email protected]'
+ )
- assert (
- email_report.render("nightly_report") == textwrap.dedent(expected_msg)
+ actual = str(email_report.render())
+ # Normalize dynamic headers
+ actual = re.sub(r'(?m)^Message-Id: <.+?>',
+ 'Message-Id: <message-id>',
+ actual)
+ actual = re.sub(r'(?m)^Date: [^\n]+ -0000$',
+ 'Date: date',
+ actual)
+ assert actual == textwrap.dedent(expected_msg)
+
+
+def test_crossbow_token_expiration_email_report(load_fixture):
+ expected_msg = load_fixture('token-expiration-email-report.txt')
+
+ class TokenExpirationReport:
+ def __init__(self, token_expiration_date, days_left):
+ self.token_expiration_date = token_expiration_date
+ self.days_left = days_left
+
+ report = TokenExpirationReport('2026-01-17', 7)
+ email_report = TokenExpirationEmailReport(
+ report=report,
+ sender_name='Sender Reporter',
+ sender_email='[email protected]',
+ recipient_email='[email protected]'
)
+ actual = str(email_report.render())
+ # Normalize dynamic headers
+ actual = re.sub(r'(?m)^Message-Id: <.+?>',
+ 'Message-Id: <message-id>',
+ actual)
+ actual = re.sub(r'(?m)^Date: [^\n]+ -0000$',
+ 'Date: date',
+ actual)
+ assert actual == textwrap.dedent(expected_msg)
+
def test_crossbow_export_report(load_fixture):
job = load_fixture('crossbow-job.yaml', decoder=yaml.load)
diff --git a/dev/archery/archery/templates/email_nightly_report.txt.j2
b/dev/archery/archery/templates/email_nightly_report.txt.j2
index bc040734b0..7b43d7c867 100644
--- a/dev/archery/archery/templates/email_nightly_report.txt.j2
+++ b/dev/archery/archery/templates/email_nightly_report.txt.j2
@@ -15,13 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#}
-{%- if True -%}
-{%- endif -%}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [NIGHTLY] Arrow Build Report for Job {{report.job.branch}}: {{
(report.tasks_by_state["error"] | length) + (report.tasks_by_state["failure"]
| length) }} failed, {{ report.tasks_by_state["pending"] | length }} pending
-
+-#}
Arrow Build Report for Job {{ report.job.branch }}
See https://s3.amazonaws.com/arrow-data/index.html for more information.
@@ -58,4 +52,4 @@ Succeeded Tasks:
- {{ task_name }}
{{ report.task_url(task) }}
{% endfor %}
-{%- endif -%}
\ No newline at end of file
+{%- endif -%}
diff --git a/dev/archery/archery/templates/email_token_expiration.txt.j2
b/dev/archery/archery/templates/email_token_expiration.txt.j2
index 54c2005e57..340cb4a535 100644
--- a/dev/archery/archery/templates/email_token_expiration.txt.j2
+++ b/dev/archery/archery/templates/email_token_expiration.txt.j2
@@ -15,12 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [CI] Arrow Crossbow Token Expiration in {{
report.token_expiration_date }}
-
+-#}
The Arrow Crossbow Token will expire in {{ report.days_left }} days.
-Please generate a new Token. Send it to Apache INFRA to update the
CROSSBOW_GITHUB_TOKEN.
-Update it on the crossbow repository and in the Azure pipelines.
+Please generate a new Token. Send it to Apache INFRA to update the
+CROSSBOW_GITHUB_TOKEN. Update it on the crossbow repository and in
+the Azure pipelines.
diff --git a/dev/archery/archery/templates/email_workflow_report.txt.j2
b/dev/archery/archery/templates/email_workflow_report.txt.j2
index 193856c180..6668d6c67e 100644
--- a/dev/archery/archery/templates/email_workflow_report.txt.j2
+++ b/dev/archery/archery/templates/email_workflow_report.txt.j2
@@ -15,13 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#}
-{%- if True -%}
-{%- endif -%}
-From: {{ sender_name }} <{{ sender_email }}>
-To: {{ recipient_email }}
-Subject: [{{ report.datetime.strftime('%Y-%m-%d') }}] Arrow Build Report for
{{ report.name }}: {{ report.failed_jobs() | length }} failed
-
+-#}
Arrow Build Report for {{ report.name }}
Workflow URL: {{ report.url }}
@@ -42,4 +36,4 @@ Succeeded Jobs:
- {{ job.name }}
{{ job.url }}
{% endfor %}
-{%- endif -%}
\ No newline at end of file
+{%- endif -%}
