This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9a66882ed5 update pattern for dataflow job id extraction (#41794)
9a66882ed5 is described below
commit 9a66882ed5bad1f08c716ecf3db10c1a6aa8406f
Author: Lukas Mikelionis <[email protected]>
AuthorDate: Sun Sep 1 03:16:26 2024 +0200
update pattern for dataflow job id extraction (#41794)
---
airflow/providers/google/cloud/hooks/dataflow.py | 2 +-
tests/providers/google/cloud/hooks/test_dataflow.py | 13 +++++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/google/cloud/hooks/dataflow.py
b/airflow/providers/google/cloud/hooks/dataflow.py
index 588d56abd5..8f7e2e2549 100644
--- a/airflow/providers/google/cloud/hooks/dataflow.py
+++ b/airflow/providers/google/cloud/hooks/dataflow.py
@@ -71,7 +71,7 @@ DEFAULT_DATAFLOW_LOCATION = "us-central1"
JOB_ID_PATTERN = re.compile(
- r"Submitted job: (?P<job_id_java>[^\"\n]*)|Created job with id:
\[(?P<job_id_python>[^\"\n]*)\]"
+ r"Submitted job: (?P<job_id_java>[^\"\n\s]*)|Created job with id:
\[(?P<job_id_python>[^\"\n\s]*)\]"
)
T = TypeVar("T", bound=Callable)
diff --git a/tests/providers/google/cloud/hooks/test_dataflow.py
b/tests/providers/google/cloud/hooks/test_dataflow.py
index 104ee374ce..4e1aa60287 100644
--- a/tests/providers/google/cloud/hooks/test_dataflow.py
+++ b/tests/providers/google/cloud/hooks/test_dataflow.py
@@ -2091,6 +2091,18 @@ INFO: To cancel the job using the 'gcloud' tool, run:
> gcloud dataflow jobs --project=XXX cancel --region=europe-west3 {TEST_JOB_ID}
"""
+# using logback pattern: [%d] %-5level %msg \(%c\) \(%t\)%n
+APACHE_BEAM_V_2_58_1_JAVA_SDK_LOG = f""""\
+[2024-08-28 08:06:39,298] INFO Dataflow SDK version: 2.58.1
(org.apache.beam.runners.dataflow.DataflowRunner) (main)
+[2024-08-28 08:06:40,305] INFO To access the Dataflow monitoring console,
please navigate to\
+https://console.cloud.google.com/dataflow/jobs/europe-west1/{TEST_JOB_ID}?project=XXXX\
+(org.apache.beam.runners.dataflow.DataflowRunner) (main)
+[2024-08-28 08:06:40,305] INFO Submitted job: {TEST_JOB_ID}
(org.apache.beam.runners.dataflow.DataflowRunner) (main)
+[2024-08-28 08:06:40,305] INFO To cancel the job using the 'gcloud' tool, run:
+> gcloud dataflow jobs --project=gowish-develop cancel --region=europe-west1
{TEST_JOB_ID}\
+(org.apache.beam.runners.dataflow.DataflowRunner) (main)
+"""
+
CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG = f"""\
WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner -
Dataflow SDK version: 2.56.0",\
"severity":"INFO"}}
@@ -2153,6 +2165,7 @@ class TestDataflow:
[
pytest.param(APACHE_BEAM_V_2_14_0_JAVA_SDK_LOG,
id="apache-beam-2.14.0-JDK"),
pytest.param(APACHE_BEAM_V_2_22_0_JAVA_SDK_LOG,
id="apache-beam-2.22.0-JDK"),
+ pytest.param(APACHE_BEAM_V_2_58_1_JAVA_SDK_LOG,
id="apache-beam-2.58.1-JDK"),
pytest.param(
CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG,
id="cloud-composer-cloud-logging-apache-beam-2.56.0-JDK",
