This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ce4a60ce27 Mark image as refreshed when pulled on CI (#23410)
ce4a60ce27 is described below
commit ce4a60ce279f691f428aa5f671180407e722ccbf
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon May 2 18:13:44 2022 +0200
Mark image as refreshed when pulled on CI (#23410)
One of the recent changes in Breeze (#23395) caused unnecessary
rebuilding of image when "build-docs" is run. On CI we build
image once and reuse it. However in case of Buld docs we missed
information that the image is "fresh" and we started rebuilding
it. This change marks the image as "refreshed" when it is pulled
with `--tag-as-latest` flag (which happens in CI).
---
.../build_image/ci/build_ci_image.py | 12 +++--
.../src/airflow_breeze/commands/common_options.py | 3 +-
dev/breeze/src/airflow_breeze/utils/pulll_image.py | 5 +-
images/breeze/output-pull-image.svg | 55 +++++++++++-----------
images/breeze/output-pull-prod-image.svg | 55 +++++++++++-----------
5 files changed, 70 insertions(+), 60 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
index 9c698a8dab..d2c12a23bf 100644
--- a/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
+++ b/dev/breeze/src/airflow_breeze/build_image/ci/build_ci_image.py
@@ -177,10 +177,7 @@ def build_ci_image(
)
if not dry_run:
if build_result.returncode == 0:
- ci_image_cache_dir = BUILD_CACHE_DIR /
ci_image_params.airflow_branch
- ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
- touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
-
calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir, update=True)
+ mark_image_as_refreshed(ci_image_params)
else:
get_console().print("[error]Error when building image![/]")
return (
@@ -194,6 +191,13 @@ def build_ci_image(
return build_result.returncode, f"Image build:
{ci_image_params.python}"
+def mark_image_as_refreshed(ci_image_params: BuildCiParams):
+ ci_image_cache_dir = BUILD_CACHE_DIR / ci_image_params.airflow_branch
+ ci_image_cache_dir.mkdir(parents=True, exist_ok=True)
+ touch_cache_file(f"built_{ci_image_params.python}",
root_dir=ci_image_cache_dir)
+ calculate_md5_checksum_for_files(ci_image_params.md5sum_cache_dir,
update=True)
+
+
def build_ci_image_in_parallel(
verbose: bool, dry_run: bool, parallelism: int, python_version_list:
List[str], **kwargs
):
diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py
b/dev/breeze/src/airflow_breeze/commands/common_options.py
index a2d2e8c0f9..2b0831e44a 100644
--- a/dev/breeze/src/airflow_breeze/commands/common_options.py
+++ b/dev/breeze/src/airflow_breeze/commands/common_options.py
@@ -294,7 +294,8 @@ option_wait_for_image = click.option(
)
option_tag_as_latest = click.option(
'--tag-as-latest',
- help='Tags the image as latest after pulling.',
+ help='Tags the image as latest and update checksum of all files after
pulling. '
+ 'Used in CI to pull the image built in another job.',
is_flag=True,
envvar='TAG_AS_LATEST',
)
diff --git a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
index 357fa1b77e..28179a2329 100644
--- a/dev/breeze/src/airflow_breeze/utils/pulll_image.py
+++ b/dev/breeze/src/airflow_breeze/utils/pulll_image.py
@@ -19,6 +19,7 @@ import multiprocessing as mp
import time
from typing import List, Tuple, Union
+from airflow_breeze.build_image.ci.build_ci_image import
mark_image_as_refreshed
from airflow_breeze.build_image.ci.build_ci_params import BuildCiParams
from airflow_breeze.build_image.prod.build_prod_params import BuildProdParams
from airflow_breeze.utils.console import get_console
@@ -80,7 +81,7 @@ def run_pull_image(
poll_time: float,
) -> Tuple[int, str]:
"""
- Pull image soecified.
+ Pull image specified.
:param image_params: Image parameters.
:param dry_run: whether it's dry run
:param verbose: whether it's verbose
@@ -138,6 +139,8 @@ def run_pull_image(
dry_run=dry_run,
check=False,
)
+ if command_result.returncode == 0 and isinstance(image_params,
BuildCiParams):
+ mark_image_as_refreshed(image_params)
return command_result.returncode, f"Image Python
{image_params.python}"
if wait_for_image:
if verbose or dry_run:
diff --git a/images/breeze/output-pull-image.svg
b/images/breeze/output-pull-image.svg
index b432347677..ed075ebadc 100644
--- a/images/breeze/output-pull-image.svg
+++ b/images/breeze/output-pull-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="934" viewBox="0 0 1720.0 934"
+<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
xmlns="http://www.w3.org/2000/svg";>
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-2708726962-terminal-wrapper span {
+ .rich-svg-921580930-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu
Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-2708726962-terminal-wrapper a {
+ .rich-svg-921580930-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-2708726962-terminal-body .blink {
- animation: rich-svg-2708726962-blinker 1s infinite;
+ .rich-svg-921580930-terminal-body .blink {
+ animation: rich-svg-921580930-blinker 1s infinite;
}
- @keyframes rich-svg-2708726962-blinker {
+ @keyframes rich-svg-921580930-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-2708726962-terminal-wrapper {
+ .rich-svg-921580930-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-2708726962-terminal {
+ .rich-svg-921580930-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-2708726962-terminal:after {
+ .rich-svg-921580930-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-2708726962-terminal-header {
+ .rich-svg-921580930-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-2708726962-terminal-title-tab {
+ .rich-svg-921580930-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-2708726962-terminal-traffic-lights {
+ .rich-svg-921580930-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-2708726962-terminal-body {
+ .rich-svg-921580930-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-2708726962-terminal-body .r1 {color: #f2f2f2;
text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r2 {font-weight: bold;color: #f2f2f2;
text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r3 {color: #e5e510; text-decoration-color:
#e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r4 {color: #7f7f7f; text-decoration-color:
#7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color:
#0c0c0c;}
-.rich-svg-2708726962-terminal-body .r5 {color: #11a8cd; text-decoration-color:
#11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r6 {color: #0dbc79; text-decoration-color:
#0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-2708726962-terminal-body .r7 {color: #78780e; text-decoration-color:
#78780e;background-color: #0c0c0c;}
+ .rich-svg-921580930-terminal-body .r1 {color: #f2f2f2;
text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r2 {font-weight: bold;color: #f2f2f2;
text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r3 {color: #e5e510; text-decoration-color:
#e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r4 {color: #7f7f7f; text-decoration-color:
#7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color:
#0c0c0c;}
+.rich-svg-921580930-terminal-body .r5 {color: #11a8cd; text-decoration-color:
#11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r6 {color: #0dbc79; text-decoration-color:
#0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-921580930-terminal-body .r7 {color: #78780e; text-decoration-color:
#78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml";>
- <div class="rich-svg-2708726962-terminal-wrapper">
- <div class="rich-svg-2708726962-terminal">
- <div class="rich-svg-2708726962-terminal-header">
- <svg
class="rich-svg-2708726962-terminal-traffic-lights" width="90" height="21"
viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg";>
+ <div class="rich-svg-921580930-terminal-wrapper">
+ <div class="rich-svg-921580930-terminal">
+ <div class="rich-svg-921580930-terminal-header">
+ <svg
class="rich-svg-921580930-terminal-traffic-lights" width="90" height="21"
viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg";>
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div
class="rich-svg-2708726962-terminal-title-tab">Command: pull-image</div>
+ <div
class="rich-svg-921580930-terminal-title-tab">Command: pull-image</div>
</div>
- <div class="rich-svg-2708726962-terminal-body">
+ <div class="rich-svg-921580930-terminal-body">
<div><span class="r2">
</span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span
class="r2">breeze pull-image [OPTIONS] [EXTRA_PYTEST_ARGS]...
</span></div>
<div><span class="r2">
</span></div>
@@ -126,7 +126,8 @@
<div><span class="r4">│</span><span class="r1">
</span><span class="r4">[default: 3.7]
</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--verify-image</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Verify image.
</span><span
class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Wait until image is
available.
</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Tags the image as latest
after pulling. </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Tags the image as latest and
update checksum of all files after pulling. Used in CI to pull </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r1">the image built in another job.
</span><span class="r1"> </span><span
class="r4">│</span></div>
<div><span
class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Parallel running
───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--run-in-parallel</span><span class="r1"> </span><span
class="r1">Run the operation in parallel on all or selected subset of Python
versions. </span><span class="r1"> </span><span
class="r4">│</span></div>
diff --git a/images/breeze/output-pull-prod-image.svg
b/images/breeze/output-pull-prod-image.svg
index a12f75146f..0253a869fa 100644
--- a/images/breeze/output-pull-prod-image.svg
+++ b/images/breeze/output-pull-prod-image.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="934" viewBox="0 0 1720.0 934"
+<svg width="1720.0" height="956" viewBox="0 0 1720.0 956"
xmlns="http://www.w3.org/2000/svg";>
<style>
@font-face {
@@ -17,30 +17,30 @@
font-style: bold;
font-weight: 700;
}
- .rich-svg-3990808719-terminal-wrapper span {
+ .rich-svg-1620392287-terminal-wrapper span {
display: inline-block;
white-space: pre;
vertical-align: top;
font-size: 18px;
font-family:'Rich Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu
Sans Mono',consolas,'Courier New',monospace;
}
- .rich-svg-3990808719-terminal-wrapper a {
+ .rich-svg-1620392287-terminal-wrapper a {
text-decoration: none;
color: inherit;
}
- .rich-svg-3990808719-terminal-body .blink {
- animation: rich-svg-3990808719-blinker 1s infinite;
+ .rich-svg-1620392287-terminal-body .blink {
+ animation: rich-svg-1620392287-blinker 1s infinite;
}
- @keyframes rich-svg-3990808719-blinker {
+ @keyframes rich-svg-1620392287-blinker {
from { opacity: 1.0; }
50% { opacity: 0.3; }
to { opacity: 1.0; }
}
- .rich-svg-3990808719-terminal-wrapper {
+ .rich-svg-1620392287-terminal-wrapper {
padding: 140px;
padding-top: 100px;
}
- .rich-svg-3990808719-terminal {
+ .rich-svg-1620392287-terminal {
position: relative;
display: flex;
flex-direction: column;
@@ -49,7 +49,7 @@
border-radius: 14px;
box-shadow: 0 0 0 1px #484848;
}
- .rich-svg-3990808719-terminal:after {
+ .rich-svg-1620392287-terminal:after {
position: absolute;
width: 100%;
height: 100%;
@@ -60,7 +60,7 @@
transform: rotate(-4.5deg);
z-index: -1;
}
- .rich-svg-3990808719-terminal-header {
+ .rich-svg-1620392287-terminal-header {
position: relative;
width: 100%;
background-color: #2e2e2e;
@@ -72,7 +72,7 @@
box-shadow: inset 0px -1px 0px 0px #4e4e4e,
inset 0px -4px 8px 0px #1a1a1a;
}
- .rich-svg-3990808719-terminal-title-tab {
+ .rich-svg-1620392287-terminal-title-tab {
display: inline-block;
margin-top: 14px;
margin-left: 124px;
@@ -85,36 +85,36 @@
inset 1px 0px 0px 0px #4e4e4e,
inset -1px 0px 0px 0px #4e4e4e;
}
- .rich-svg-3990808719-terminal-traffic-lights {
+ .rich-svg-1620392287-terminal-traffic-lights {
position: absolute;
top: 24px;
left: 20px;
}
- .rich-svg-3990808719-terminal-body {
+ .rich-svg-1620392287-terminal-body {
line-height: 22px;
padding: 14px;
}
- .rich-svg-3990808719-terminal-body .r1 {color: #f2f2f2;
text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r2 {font-weight: bold;color: #f2f2f2;
text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r3 {color: #e5e510; text-decoration-color:
#e5e510; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r4 {color: #7f7f7f; text-decoration-color:
#7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color:
#0c0c0c;}
-.rich-svg-3990808719-terminal-body .r5 {color: #11a8cd; text-decoration-color:
#11a8cd; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r6 {color: #0dbc79; text-decoration-color:
#0dbc79; font-weight: bold;background-color: #0c0c0c;}
-.rich-svg-3990808719-terminal-body .r7 {color: #78780e; text-decoration-color:
#78780e;background-color: #0c0c0c;}
+ .rich-svg-1620392287-terminal-body .r1 {color: #f2f2f2;
text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r2 {font-weight: bold;color: #f2f2f2;
text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r3 {color: #e5e510; text-decoration-color:
#e5e510; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r4 {color: #7f7f7f; text-decoration-color:
#7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color:
#0c0c0c;}
+.rich-svg-1620392287-terminal-body .r5 {color: #11a8cd; text-decoration-color:
#11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r6 {color: #0dbc79; text-decoration-color:
#0dbc79; font-weight: bold;background-color: #0c0c0c;}
+.rich-svg-1620392287-terminal-body .r7 {color: #78780e; text-decoration-color:
#78780e;background-color: #0c0c0c;}
</style>
<foreignObject x="0" y="0" width="100%" height="100%">
<body xmlns="http://www.w3.org/1999/xhtml";>
- <div class="rich-svg-3990808719-terminal-wrapper">
- <div class="rich-svg-3990808719-terminal">
- <div class="rich-svg-3990808719-terminal-header">
- <svg
class="rich-svg-3990808719-terminal-traffic-lights" width="90" height="21"
viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg";>
+ <div class="rich-svg-1620392287-terminal-wrapper">
+ <div class="rich-svg-1620392287-terminal">
+ <div class="rich-svg-1620392287-terminal-header">
+ <svg
class="rich-svg-1620392287-terminal-traffic-lights" width="90" height="21"
viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg";>
<circle cx="14" cy="8" r="8" fill="#ff6159"/>
<circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
<circle cx="62" cy="8" r="8" fill="#28c941"/>
</svg>
- <div
class="rich-svg-3990808719-terminal-title-tab">Command: pull-prod-image</div>
+ <div
class="rich-svg-1620392287-terminal-title-tab">Command: pull-prod-image</div>
</div>
- <div class="rich-svg-3990808719-terminal-body">
+ <div class="rich-svg-1620392287-terminal-body">
<div><span class="r2">
</span></div>
<div><span class="r2"> </span><span class="r3">Usage: </span><span
class="r2">breeze pull-prod-image [OPTIONS] [EXTRA_PYTEST_ARGS]...
</span></div>
<div><span class="r2">
</span></div>
@@ -126,7 +126,8 @@
<div><span class="r4">│</span><span class="r1">
</span><span class="r4">[default: 3.7]
</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--verify-image</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Verify image.
</span><span
class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--wait-for-image</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Wait until image is
available.
</span><span class="r1"> </span><span class="r4">│</span></div>
-<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Tags the image as latest
after pulling. </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--tag-as-latest</span><span class="r1"> </span><span class="r1">
</span><span class="r1"> </span><span class="r1">Tags the image as latest and
update checksum of all files after pulling. Used in CI to pull </span><span
class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1">
</span><span class="r1">the image built in another job.
</span><span class="r1"> </span><span
class="r4">│</span></div>
<div><span
class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
<div><span class="r4">╭─ Parallel running
───────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span
class="r5">--run-in-parallel</span><span class="r1"> </span><span
class="r1">Run the operation in parallel on all or selected subset of Python
versions. </span><span class="r1"> </span><span
class="r4">│</span></div>
