stephen-lazarionok commented on issue #39880:
URL: https://github.com/apache/airflow/issues/39880#issuecomment-2134520818
Hi, here is the codebase:
```
from datetime import datetime, timedelta
from airflow import DAG
from airflow.decorators import task
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import (
KubernetesPodOperator,
)
from dags.config import (
IMAGE_PULL_SECRET_NAME,
MISC_IMAGE_NAME,
MISC_ROUTINES_MOST_RECENT_TAG,
NAMESPACE,
REGISTRY_NAME,
misc_jobs_args,
)
from dags.constants import JobType, Platform
from dags.utils import failure_callback, get_job_name
from kubernetes.client import models as k8s
K8S_ARGS = {}
JOB_NAME = "MY_JOB_NAME"
with DAG(
dag_id=JOB_NAME,
default_args=misc_jobs_args,
catchup=False,
max_active_runs=1,
is_paused_upon_creation=False,
schedule_interval="0 * * * *", # Run once an hour at the beginning of
the hour
start_date=datetime(2022, 7, 13, 11, 0),
dagrun_timeout=timedelta(minutes=10),
on_failure_callback=failure_callback,
tags=[
f"job_type:{JobType.ETL}",
f"platform:{Platform.ALL_PLATFORMS}",
],
) as dag:
read_groups = KubernetesPodOperator(
namespace=NAMESPACE,
image=f"{REGISTRY_NAME}/{MISC_IMAGE_NAME}:{MISC_ROUTINES_MOST_RECENT_TAG}",
cmds=["python"],
arguments=["/src/pipelines/read_groups.py"],
name="read_groups",
task_id="read_groups",
get_logs=True,
is_delete_operator_pod=True,
do_xcom_push=True,
dag=dag,
labels={
"job_type": JobType.ETL,
"platform": Platform.ALL_PLATFORMS,
"app": JOB_NAME,
},
container_resources=k8s.V1ResourceRequirements(
limits={"memory": "256Mi"}, requests={"cpu": "100m", "memory":
"256Mi"}
),
image_pull_secrets=[k8s.V1LocalObjectReference(IMAGE_PULL_SECRET_NAME)],
env_vars=[
k8s.V1EnvVar(
name="PLATFORM_NAME",
value="sv",
),
k8s.V1EnvVar(
name="ML_MODEL_ENTRY_NAME",
value="my_model",
),
],
)
KubernetesPodOperator.partial(
namespace=NAMESPACE,
image=f"{REGISTRY_NAME}/{MISC_IMAGE_NAME}:{MISC_ROUTINES_MOST_RECENT_TAG}",
cmds=["python"],
name="process_group",
task_id="process_group",
get_logs=True,
is_delete_operator_pod=True,
dag=dag,
labels={
"job_type": JobType.ETL,
"platform": Platform.ALL_PLATFORMS,
"app": JOB_NAME,
},
container_resources=k8s.V1ResourceRequirements(
limits={"memory": "256Mi"}, requests={"cpu": "100m", "memory":
"256Mi"}
),
image_pull_secrets=[k8s.V1LocalObjectReference(IMAGE_PULL_SECRET_NAME)],
**K8S_ARGS,
).expand(
arguments=[
["/src/pipelines/sampleprocess.py"],
["/src/pipelines/sampleprocess.py"],
]
)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
