(airflow) branch main updated: Test connection to Qdrant (#39273)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0c96b06204 Test connection to Qdrant (#39273)
0c96b06204 is described below
commit 0c96b062044a96bcfb83991a4f4fac8e14c0bb57
Author: Anush
AuthorDate: Fri Apr 26 12:21:12 2024 +0530
Test connection to Qdrant (#39273)
---
airflow/providers/qdrant/hooks/qdrant.py | 4
airflow/providers/qdrant/provider.yaml | 2 +-
generated/provider_dependencies.json | 2 +-
scripts/ci/docker-compose/integration-qdrant.yml | 2 +-
tests/integration/providers/qdrant/operators/test_qdrant_ingest.py | 2 +-
5 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/airflow/providers/qdrant/hooks/qdrant.py
b/airflow/providers/qdrant/hooks/qdrant.py
index 31aa0b5e1d..0e501b8ac6 100644
--- a/airflow/providers/qdrant/hooks/qdrant.py
+++ b/airflow/providers/qdrant/hooks/qdrant.py
@@ -126,3 +126,7 @@ class QdrantHook(BaseHook):
return True, "Connection established!"
except (UnexpectedResponse, RpcError, ValueError) as e:
return False, str(e)
+
+def test_connection(self) -> tuple[bool, str]:
+"""Test the connection to the Qdrant instance."""
+return self.verify_connection()
diff --git a/airflow/providers/qdrant/provider.yaml
b/airflow/providers/qdrant/provider.yaml
index 7d62ed57e6..7a95d58ae6 100644
--- a/airflow/providers/qdrant/provider.yaml
+++ b/airflow/providers/qdrant/provider.yaml
@@ -38,7 +38,7 @@ integrations:
tags: [software]
dependencies:
- - qdrant_client>=1.7.0
+ - qdrant_client>=1.9.0
- apache-airflow>=2.7.0
hooks:
diff --git a/generated/provider_dependencies.json
b/generated/provider_dependencies.json
index a95bcd312e..d91b488580 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -950,7 +950,7 @@
"qdrant": {
"deps": [
"apache-airflow>=2.7.0",
- "qdrant_client>=1.7.0"
+ "qdrant_client>=1.9.0"
],
"devel-deps": [],
"cross-providers-deps": [],
diff --git a/scripts/ci/docker-compose/integration-qdrant.yml
b/scripts/ci/docker-compose/integration-qdrant.yml
index 724c1e92cf..1cafac732d 100644
--- a/scripts/ci/docker-compose/integration-qdrant.yml
+++ b/scripts/ci/docker-compose/integration-qdrant.yml
@@ -18,7 +18,7 @@
version: "3.8"
services:
qdrant:
-image: qdrant/qdrant:latest
+image: qdrant/qdrant:v1.9.0
labels:
breeze.description: "Integration required for Qdrant tests."
ports:
diff --git a/tests/integration/providers/qdrant/operators/test_qdrant_ingest.py
b/tests/integration/providers/qdrant/operators/test_qdrant_ingest.py
index eed4bd8e78..534622ec90 100644
--- a/tests/integration/providers/qdrant/operators/test_qdrant_ingest.py
+++ b/tests/integration/providers/qdrant/operators/test_qdrant_ingest.py
@@ -60,7 +60,7 @@ class TestQdrantIngestOperator:
hook = operator.hook
-hook.conn.recreate_collection(
+hook.conn.create_collection(
collection_name, vectors_config=VectorParams(size=dimensions,
distance=Distance.COSINE)
)
(airflow) branch main updated (e5bf91b7af -> 0759e6f891)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from e5bf91b7af Add stacklevel into the `suppress_and_warn` warning (#39263) add 0759e6f891 Remove deprecation warnings in `db migrate` tests (#39269) No new revisions were added by this update. Summary of changes: tests/cli/commands/test_db_command.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(airflow) branch main updated (adeb7f7cba -> e5bf91b7af)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from adeb7f7cba Fix trigger kwarg encryption migration (#39246) add e5bf91b7af Add stacklevel into the `suppress_and_warn` warning (#39263) No new revisions were added by this update. Summary of changes: airflow/providers/jdbc/hooks/jdbc.py| 6 +- pyproject.toml | 1 - tests/providers/jdbc/hooks/test_jdbc.py | 13 ++--- 3 files changed, 11 insertions(+), 9 deletions(-)
(airflow) branch main updated: add dns_config and diagnostics parameters to AzureContainerInstancesOperator (#39156)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new df1f75b676 add dns_config and diagnostics parameters to
AzureContainerInstancesOperator (#39156)
df1f75b676 is described below
commit df1f75b676a4e294d5d89ba590416f26c937efc1
Author: Kyle Andrews <90274941+kylea...@users.noreply.github.com>
AuthorDate: Thu Apr 25 19:44:03 2024 +0100
add dns_config and diagnostics parameters to
AzureContainerInstancesOperator (#39156)
* add dns_config and diagnostics parameters
* add unit tests. revert changelog update.
---
.../azure/operators/container_instances.py | 17
.../azure/operators/test_container_instances.py| 50 ++
2 files changed, 67 insertions(+)
diff --git a/airflow/providers/microsoft/azure/operators/container_instances.py
b/airflow/providers/microsoft/azure/operators/container_instances.py
index deb5579f14..6bc64e4547 100644
--- a/airflow/providers/microsoft/azure/operators/container_instances.py
+++ b/airflow/providers/microsoft/azure/operators/container_instances.py
@@ -25,8 +25,10 @@ from typing import TYPE_CHECKING, Any, Sequence
from azure.mgmt.containerinstance.models import (
Container,
ContainerGroup,
+ContainerGroupDiagnostics,
ContainerGroupSubnetId,
ContainerPort,
+DnsConfiguration,
EnvironmentVariable,
IpAddress,
ResourceRequests,
@@ -90,6 +92,8 @@ class AzureContainerInstancesOperator(BaseOperator):
Possible values include: 'Always', 'OnFailure', 'Never'
:param ip_address: The IP address type of the container group.
:param subnet_ids: The subnet resource IDs for a container group
+:param dns_config: The DNS configuration for a container group.
+:param diagnostics: Container group diagnostic information (Log Analytics).
**Example**::
@@ -113,6 +117,13 @@ class AzureContainerInstancesOperator(BaseOperator):
memory_in_gb=14.0,
cpu=4.0,
gpu=GpuResource(count=1, sku="K80"),
+dns_config=["10.0.0.10", "10.0.0.11"],
+diagnostics={
+"log_analytics": {
+"workspaceId": "workspaceid",
+"workspaceKey": "workspaceKey",
+}
+},
command=["/bin/echo", "world"],
task_id="start_container",
)
@@ -145,6 +156,8 @@ class AzureContainerInstancesOperator(BaseOperator):
ip_address: IpAddress | None = None,
ports: list[ContainerPort] | None = None,
subnet_ids: list[ContainerGroupSubnetId] | None = None,
+dns_config: DnsConfiguration | None = None,
+diagnostics: ContainerGroupDiagnostics | None = None,
**kwargs,
) -> None:
super().__init__(**kwargs)
@@ -183,6 +196,8 @@ class AzureContainerInstancesOperator(BaseOperator):
self.ip_address = ip_address
self.ports = ports
self.subnet_ids = subnet_ids
+self.dns_config = dns_config
+self.diagnostics = diagnostics
def execute(self, context: Context) -> int:
# Check name again in case it was templated.
@@ -256,6 +271,8 @@ class AzureContainerInstancesOperator(BaseOperator):
tags=self.tags,
ip_address=self.ip_address,
subnet_ids=self.subnet_ids,
+dns_config=self.dns_config,
+diagnostics=self.diagnostics,
)
self._ci_hook.create_or_update(self.resource_group, self.name,
container_group)
diff --git
a/tests/providers/microsoft/azure/operators/test_container_instances.py
b/tests/providers/microsoft/azure/operators/test_container_instances.py
index 6205ac6f5d..0a153e2fdd 100644
--- a/tests/providers/microsoft/azure/operators/test_container_instances.py
+++ b/tests/providers/microsoft/azure/operators/test_container_instances.py
@@ -410,3 +410,53 @@ class TestACIOperator:
aci.execute(None)
assert log_mock.call_count == 0
+
+
@mock.patch("airflow.providers.microsoft.azure.operators.container_instances.AzureContainerInstanceHook")
+def test_execute_with_dnsconfig(self, aci_mock):
+dns_config = MagicMock()
+
+aci_mock.return_value.get_state.return_value = make_mock_container(
+state="Terminated", exit_code=0, detail_status="test"
+)
+aci_mock.return_value.exists.return_value = False
+
+aci = AzureContainerInstancesOperator(
+ci_conn_id=None,
+registry_conn_id=None,
+resource_group="resource-group",
+name="container-name",
+i
(airflow) branch main updated: Get served logs when remote or executor logs not available for non-running task try (#39177)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new eca077b3b9 Get served logs when remote or executor logs not available
for non-running task try (#39177)
eca077b3b9 is described below
commit eca077b3b994813a09942497704ec61e35efd7d5
Author: Kalle Ahlström <71292737+kahls...@users.noreply.github.com>
AuthorDate: Thu Apr 25 16:19:37 2024 +0300
Get served logs when remote or executor logs not available for non-running
task try (#39177)
---
airflow/utils/log/file_task_handler.py | 12 ++--
tests/utils/test_log_handlers.py | 14 --
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/airflow/utils/log/file_task_handler.py
b/airflow/utils/log/file_task_handler.py
index 72b8deedbb..2a1dfd25f6 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -366,10 +366,7 @@ class FileTaskHandler(logging.Handler):
executor_messages: list[str] = []
executor_logs: list[str] = []
served_logs: list[str] = []
-is_running = ti.try_number == try_number and ti.state in (
-TaskInstanceState.RUNNING,
-TaskInstanceState.DEFERRED,
-)
+is_in_running_or_deferred = ti.state in (TaskInstanceState.RUNNING,
TaskInstanceState.DEFERRED)
with suppress(NotImplementedError):
remote_messages, remote_logs = self._read_remote_logs(ti,
try_number, metadata)
messages_list.extend(remote_messages)
@@ -384,7 +381,9 @@ class FileTaskHandler(logging.Handler):
worker_log_full_path = Path(self.local_base, worker_log_rel_path)
local_messages, local_logs =
self._read_from_local(worker_log_full_path)
messages_list.extend(local_messages)
-if is_running and not executor_messages:
+if is_in_running_or_deferred and not executor_messages and not
remote_logs:
+# While task instance is still running and we don't have either
executor nor remote logs, look for served logs
+# This is for cases when users have not setup remote logging nor
shared drive for logs
served_messages, served_logs = self._read_from_logs_server(ti,
worker_log_rel_path)
messages_list.extend(served_messages)
elif ti.state not in State.unfinished and not (local_logs or
remote_logs):
@@ -404,11 +403,12 @@ class FileTaskHandler(logging.Handler):
)
log_pos = len(logs)
messages = "".join([f"*** {x}\n" for x in messages_list])
+end_of_log = ti.try_number != try_number or not
is_in_running_or_deferred
if metadata and "log_pos" in metadata:
previous_chars = metadata["log_pos"]
logs = logs[previous_chars:] # Cut off previously passed log test
as new tail
out_message = logs if "log_pos" in (metadata or {}) else messages +
logs
-return out_message, {"end_of_log": not is_running, "log_pos": log_pos}
+return out_message, {"end_of_log": end_of_log, "log_pos": log_pos}
@staticmethod
def _get_pod_namespace(ti: TaskInstance):
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index ba849825d5..2bfc574b64 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -315,7 +315,7 @@ class TestFileTaskLogHandler:
def test__read_for_celery_executor_fallbacks_to_worker(self,
create_task_instance):
"""Test for executors which do not have `get_task_log` method, it
fallbacks to reading
-log from worker. But it happens only for the latest try_number."""
+log from worker if and only if remote logs aren't found"""
executor_name = "CeleryExecutor"
ti = create_task_instance(
@@ -336,7 +336,17 @@ class TestFileTaskLogHandler:
fth._read_from_logs_server.assert_called_once()
assert actual == ("*** this message\nthis\nlog\ncontent",
{"end_of_log": False, "log_pos": 16})
-# Previous try_number is from remote logs without reaching worker
server
+# Previous try_number should return served logs when remote logs
aren't implemented
+fth._read_from_logs_server = mock.Mock()
+fth._read_from_logs_server.return_value = ["served logs
try_number=1"], ["this\nlog\ncontent"]
+actual = fth._read(ti=ti, try_number=1)
+fth._read_from_logs_server.assert_called_once()
+assert actual == (
+"*** served logs try_number=1\nthis\nlog\ncontent",
+{"end_of
(airflow) branch main updated: Implement run-method on KiotaRequestAdapterHook and move logic away from triggerer to hook (#39237)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 15c2734c4c Implement run-method on KiotaRequestAdapterHook and move
logic away from triggerer to hook (#39237)
15c2734c4c is described below
commit 15c2734c4cac2e2acbdeaf28dfe67d04e28cee90
Author: David Blain
AuthorDate: Thu Apr 25 13:59:55 2024 +0200
Implement run-method on KiotaRequestAdapterHook and move logic away from
triggerer to hook (#39237)
-
Co-authored-by: David Blain
---
airflow/providers/microsoft/azure/hooks/msgraph.py | 164 -
.../providers/microsoft/azure/triggers/msgraph.py | 98 ++--
.../microsoft/azure/hooks/test_msgraph.py | 64 +++-
.../microsoft/azure/triggers/test_msgraph.py | 28
4 files changed, 231 insertions(+), 123 deletions(-)
diff --git a/airflow/providers/microsoft/azure/hooks/msgraph.py
b/airflow/providers/microsoft/azure/hooks/msgraph.py
index 7fcc328f86..84b2252bd2 100644
--- a/airflow/providers/microsoft/azure/hooks/msgraph.py
+++ b/airflow/providers/microsoft/azure/hooks/msgraph.py
@@ -17,28 +17,74 @@
# under the License.
from __future__ import annotations
-from typing import TYPE_CHECKING
-from urllib.parse import urljoin, urlparse
+import json
+from http import HTTPStatus
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, Callable
+from urllib.parse import quote, urljoin, urlparse
import httpx
from azure.identity import ClientSecretCredential
from httpx import Timeout
+from kiota_abstractions.api_error import APIError
+from kiota_abstractions.method import Method
+from kiota_abstractions.request_information import RequestInformation
+from kiota_abstractions.response_handler import ResponseHandler
from kiota_authentication_azure.azure_identity_authentication_provider import (
AzureIdentityAuthenticationProvider,
)
from kiota_http.httpx_request_adapter import HttpxRequestAdapter
-from msgraph_core import GraphClientFactory
-from msgraph_core._enums import APIVersion, NationalClouds
+from kiota_http.middleware.options import ResponseHandlerOption
+from msgraph_core import APIVersion, GraphClientFactory
+from msgraph_core._enums import NationalClouds
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowBadRequest, AirflowException,
AirflowNotFoundException
from airflow.hooks.base import BaseHook
if TYPE_CHECKING:
from kiota_abstractions.request_adapter import RequestAdapter
+from kiota_abstractions.request_information import QueryParams
+from kiota_abstractions.response_handler import NativeResponseType
+from kiota_abstractions.serialization import ParsableFactory
+from kiota_http.httpx_request_adapter import ResponseType
from airflow.models import Connection
+class CallableResponseHandler(ResponseHandler):
+"""
+CallableResponseHandler executes the passed callable_function with
response as parameter.
+
+param callable_function: Function that is applied to the response.
+"""
+
+def __init__(
+self,
+callable_function: Callable[[NativeResponseType, dict[str,
ParsableFactory | None] | None], Any],
+):
+self.callable_function = callable_function
+
+async def handle_response_async(
+self, response: NativeResponseType, error_map: dict[str,
ParsableFactory | None] | None = None
+) -> Any:
+"""
+Invoke this callback method when a response is received.
+
+param response: The type of the native response object.
+param error_map: The error dict to use in case of a failed request.
+"""
+value = self.callable_function(response, error_map)
+if response.status_code not in {200, 201, 202, 204, 302}:
+message = value or response.reason_phrase
+status_code = HTTPStatus(response.status_code)
+if status_code == HTTPStatus.BAD_REQUEST:
+raise AirflowBadRequest(message)
+elif status_code == HTTPStatus.NOT_FOUND:
+raise AirflowNotFoundException(message)
+raise AirflowException(message)
+return value
+
+
class KiotaRequestAdapterHook(BaseHook):
"""
A Microsoft Graph API interaction hook, a Wrapper around
KiotaRequestAdapter.
@@ -54,6 +100,7 @@ class KiotaRequestAdapterHook(BaseHook):
or you can pass a string as "v1.0" or "beta".
"""
+DEFAULT_HEADERS = {"Accept": "application/json;q=1"}
cached_request_adapters: dict[str, tuple[APIVersion, RequestAdapter]] = {}
default_conn_name: str = "msgraph_default"
@@ -117,13 +164,16 @@ class KiotaRequestAd
(airflow) branch main updated: Capture warning during setup and collect tests cases (#39250)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 5eaf17339b Capture warning during setup and collect tests cases
(#39250)
5eaf17339b is described below
commit 5eaf17339b9a80e944ce2ea2502cd31babeb8bb3
Author: Andrey Anshin
AuthorDate: Thu Apr 25 15:32:43 2024 +0400
Capture warning during setup and collect tests cases (#39250)
---
contributing-docs/testing/unit_tests.rst | 9 ++-
scripts/ci/testing/summarize_captured_warnings.py | 49 ++
tests/_internals/capture_warnings.py | 82 +--
3 files changed, 91 insertions(+), 49 deletions(-)
diff --git a/contributing-docs/testing/unit_tests.rst
b/contributing-docs/testing/unit_tests.rst
index 9d1dac3ba1..e9a7263a4e 100644
--- a/contributing-docs/testing/unit_tests.rst
+++ b/contributing-docs/testing/unit_tests.rst
@@ -1152,10 +1152,13 @@ or by setting the environment variable
``CAPTURE_WARNINGS_OUTPUT``.
root@3f98e75b1ebe:/opt/airflow# pytest tests/core/
--warning-output-path=/foo/bar/spam.egg
...
-= Warning summary. Total: 34, Unique: 16
==
+= Warning summary. Total: 28, Unique: 12
==
airflow: total 11, unique 1
-other: total 12, unique 4
-tests: total 11, unique 11
+ runtest: total 11, unique 1
+other: total 7, unique 1
+ runtest: total 7, unique 1
+tests: total 10, unique 10
+ runtest: total 10, unique 10
Warnings saved into /foo/bar/spam.egg file.
= short test summary info
=
diff --git a/scripts/ci/testing/summarize_captured_warnings.py
b/scripts/ci/testing/summarize_captured_warnings.py
index 2513aad6fa..3187d9ab9c 100755
--- a/scripts/ci/testing/summarize_captured_warnings.py
+++ b/scripts/ci/testing/summarize_captured_warnings.py
@@ -37,7 +37,7 @@ if __name__ not in ("__main__", "__mp_main__"):
)
-REQUIRED_FIELDS = ("category", "message", "node_id", "filename", "lineno",
"group", "count")
+REQUIRED_FIELDS = ("category", "message", "node_id", "filename", "lineno",
"group", "count", "when")
CONSOLE_SIZE = shutil.get_terminal_size((80, 20)).columns
# Use as prefix/suffix in report output
IMPORTANT_WARNING_SIGN = {
@@ -71,8 +71,8 @@ WARNINGS_BAD = warnings_filename("bad")
@functools.lru_cache(maxsize=None)
-def _unique_key(*args: str) -> str:
-return str(uuid5(NAMESPACE_OID, "-".join(args)))
+def _unique_key(*args: str | None) -> str:
+return str(uuid5(NAMESPACE_OID, "-".join(map(str, args
def sorted_groupby(it, grouping_key: Callable):
@@ -95,9 +95,10 @@ def count_groups(
class CapturedWarnings:
category: str
message: str
-node_id: str
filename: str
lineno: int
+when: str
+node_id: str | None
@property
def unique_warning(self) -> str:
@@ -176,8 +177,8 @@ def merge_files(files: Iterator[tuple[Path, str]],
output_directory: Path) -> Pa
return output_file
-def group_report_warnings(group, group_records, output_directory: Path) ->
None:
-output_filepath = output_directory / warnings_filename(f"group-{group}")
+def group_report_warnings(group, when: str, group_records, output_directory:
Path) -> None:
+output_filepath = output_directory / warnings_filename(f"{group}-{when}")
group_warnings: dict[str, CapturedWarnings] = {}
unique_group_warnings: dict[str, CapturedWarnings] = {}
@@ -188,27 +189,21 @@ def group_report_warnings(group, group_records,
output_directory: Path) -> None:
if cw.unique_warning not in unique_group_warnings:
unique_group_warnings[cw.unique_warning] = cw
-print(f" Group {group!r} ".center(CONSOLE_SIZE, "="))
+print(f" Group {group!r} on {when!r} ".center(CONSOLE_SIZE, "="))
with output_filepath.open(mode="w") as fp:
for cw in group_warnings.values():
fp.write(f"{cw.output()}\n")
print(f"Saved into file: {output_filepath.as_posix()}\n")
-print(f"Unique warnings within the test cases: {len(group_warnings):,}\n")
-print("Top 10 Tests Cases:")
-it = count_groups(
-group_warnings.values(),
-grouping_key=lambda cw: (
-cw.category,
-cw.node_id,
-),
-top=10,
-)
-for (category, node_id), count in it:
-if suffix := IMPORTANT_WARNING_SIGN.get(category, ""):
-
(airflow) branch main updated: Remove unnecessary validation from cncf provider. (#39238)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4920ab25b3 Remove unnecessary validation from cncf provider. (#39238)
4920ab25b3 is described below
commit 4920ab25b3062c04222823f3c47b8d4d8be7bd97
Author: V.Shkaberda
AuthorDate: Thu Apr 25 10:16:22 2024 +0300
Remove unnecessary validation from cncf provider. (#39238)
Co-authored-by: Шкаберда Вадим Миколайович
---
.../kubernetes/operators/custom_object_launcher.py | 3 +--
.../operators/test_custom_object_launcher.py | 28 ++
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git
a/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py
b/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py
index 439c51e3cb..77d99a0fba 100644
--- a/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py
+++ b/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py
@@ -60,12 +60,11 @@ class SparkJobSpec:
if self.spec.get("dynamicAllocation", {}).get("enabled"):
if not all(
[
-self.spec["dynamicAllocation"].get("initialExecutors"),
self.spec["dynamicAllocation"].get("minExecutors"),
self.spec["dynamicAllocation"].get("maxExecutors"),
]
):
-raise AirflowException("Make sure initial/min/max value for
dynamic allocation is passed")
+raise AirflowException("Make sure min/max value for dynamic
allocation is passed")
def update_resources(self):
if self.spec["driver"].get("container_resources"):
diff --git
a/tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py
b/tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py
index d33fdd6048..3a57fdefdb 100644
--- a/tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py
+++ b/tests/providers/cncf/kubernetes/operators/test_custom_object_launcher.py
@@ -64,6 +64,22 @@ class TestSparkJobSpec:
assert spark_job_spec.spec["dynamicAllocation"]["enabled"]
+def
test_spark_job_spec_dynamicAllocation_enabled_with_default_initial_executors(self):
+entries = {
+"spec": {
+"dynamicAllocation": {
+"enabled": True,
+"minExecutors": 1,
+"maxExecutors": 2,
+},
+"driver": {},
+"executor": {},
+}
+}
+spark_job_spec = SparkJobSpec(**entries)
+
+assert spark_job_spec.spec["dynamicAllocation"]["enabled"]
+
def
test_spark_job_spec_dynamicAllocation_enabled_with_invalid_config(self):
entries = {
"spec": {
@@ -78,19 +94,11 @@ class TestSparkJobSpec:
}
}
-cloned_entries = entries.copy()
-cloned_entries["spec"]["dynamicAllocation"]["initialExecutors"] = None
-with pytest.raises(
-AirflowException,
-match="Make sure initial/min/max value for dynamic allocation is
passed",
-):
-SparkJobSpec(**cloned_entries)
-
cloned_entries = entries.copy()
cloned_entries["spec"]["dynamicAllocation"]["minExecutors"] = None
with pytest.raises(
AirflowException,
-match="Make sure initial/min/max value for dynamic allocation is
passed",
+match="Make sure min/max value for dynamic allocation is passed",
):
SparkJobSpec(**cloned_entries)
@@ -98,7 +106,7 @@ class TestSparkJobSpec:
cloned_entries["spec"]["dynamicAllocation"]["maxExecutors"] = None
with pytest.raises(
AirflowException,
-match="Make sure initial/min/max value for dynamic allocation is
passed",
+match="Make sure min/max value for dynamic allocation is passed",
):
SparkJobSpec(**cloned_entries)
(airflow) branch main updated (12ce2dcd46 -> 9619536e6f)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 12ce2dcd46 Retry merge commit retrieval on failure (#39231) add 9619536e6f Allow trust env parameter to be defined in extra options of HTTP Connection (#39161) No new revisions were added by this update. Summary of changes: airflow/providers/http/hooks/http.py| 4 tests/providers/http/hooks/test_http.py | 23 +++ 2 files changed, 27 insertions(+)
(airflow) branch main updated: Retry merge commit retrieval on failure (#39231)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 12ce2dcd46 Retry merge commit retrieval on failure (#39231)
12ce2dcd46 is described below
commit 12ce2dcd46f6ad59e0a03925654a92126c2e5164
Author: Jarek Potiuk
AuthorDate: Wed Apr 24 19:49:22 2024 +0200
Retry merge commit retrieval on failure (#39231)
Sometimes when merge commit retrieval fails (because of race condition
most likely as pull request event is no ready yet) build-images builds
all images because selective check does not know the merge commit (in
pull request target vuild image failure). That was the only side effect.
because otherwise build image had a fallback to
github.event.pull_request.head.sha in this case.
With this PR, we will retry it and fallback explicitly to
github.event.pull_request.head.sha if it does not work.
---
.github/workflows/build-images.yml | 19 ++-
1 file changed, 18 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 9c732d1166..bd10e73aac 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -91,8 +91,25 @@ jobs:
- name: Discover PR merge commit
id: discover-pr-merge-commit
run: |
+ # Sometimes target-commit-sha cannot be
TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}'
--jq .merge_commit_sha)"
- echo "TARGET_COMMIT_SHA=$TARGET_COMMIT_SHA" >> ${GITHUB_ENV}
+ if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
+# Sometimes retrieving the merge commit SHA from PR fails. We
retry it once. Otherwise we
+# fall-back to github.event.pull_request.head.sha
+echo
+echo "Could not retrieve merge commit SHA from PR, waiting for 3
seconds and retrying."
+echo
+sleep 3
+TARGET_COMMIT_SHA="$(gh api '${{ github.event.pull_request.url }}'
--jq .merge_commit_sha)"
+if [[ ${TARGET_COMMIT_SHA} == "" ]]; then
+ echo
+ echo "Could not retrieve merge commit SHA from PR, falling back
to PR head SHA."
+ echo
+ TARGET_COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
+fi
+ fi
+ echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}"
+ echo "TARGET_COMMIT_SHA=${TARGET_COMMIT_SHA}" >> ${GITHUB_ENV}
echo "target-commit-sha=${TARGET_COMMIT_SHA}" >> ${GITHUB_OUTPUT}
if: github.event_name == 'pull_request_target'
# The labels in the event aren't updated when re-triggering the job, So
lets hit the API to get
(airflow) branch main updated: Fixed side effect of menu filtering causing disappearing menus (#39229)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0d2c0c5cf0 Fixed side effect of menu filtering causing disappearing
menus (#39229)
0d2c0c5cf0 is described below
commit 0d2c0c5cf04ef886a8211820d0dc2f4cd8c47251
Author: Jarek Potiuk
AuthorDate: Wed Apr 24 19:09:09 2024 +0200
Fixed side effect of menu filtering causing disappearing menus (#39229)
The default implementation of filter_permitted_menu_items had a
subtle side-effect. The filtering on child items was done in-place
and modified the menu itself, so it was enough to get the same
worker serve requests for multiple users for the same menu to get
the items removed for subsequent user - even if they had permission
to see it.
Deepcopying the menu items before filtering them should fix the problem
Fixes: #39204
Fixes: #39135
---
airflow/auth/managers/base_auth_manager.py| 14 --
tests/auth/managers/test_base_auth_manager.py | 40 +++
2 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/airflow/auth/managers/base_auth_manager.py
b/airflow/auth/managers/base_auth_manager.py
index 7bb4e92889..44fc53a66e 100644
--- a/airflow/auth/managers/base_auth_manager.py
+++ b/airflow/auth/managers/base_auth_manager.py
@@ -21,6 +21,7 @@ from abc import abstractmethod
from functools import cached_property
from typing import TYPE_CHECKING, Container, Literal, Sequence
+from flask_appbuilder.menu import MenuItem
from sqlalchemy import select
from airflow.auth.managers.models.resource_details import (
@@ -34,7 +35,6 @@ from airflow.utils.session import NEW_SESSION, provide_session
if TYPE_CHECKING:
from flask import Blueprint
-from flask_appbuilder.menu import MenuItem
from sqlalchemy.orm import Session
from airflow.auth.managers.models.base_user import BaseUser
@@ -397,13 +397,21 @@ class BaseAuthManager(LoggingMixin):
)
accessible_items = []
for menu_item in items:
+menu_item_copy = MenuItem(
+name=menu_item.name,
+icon=menu_item.icon,
+label=menu_item.label,
+childs=[],
+baseview=menu_item.baseview,
+cond=menu_item.cond,
+)
if menu_item.childs:
accessible_children = []
for child in menu_item.childs:
if
self.security_manager.has_access(ACTION_CAN_ACCESS_MENU, child.name):
accessible_children.append(child)
-menu_item.childs = accessible_children
-accessible_items.append(menu_item)
+menu_item_copy.childs = accessible_children
+accessible_items.append(menu_item_copy)
return accessible_items
@abstractmethod
diff --git a/tests/auth/managers/test_base_auth_manager.py
b/tests/auth/managers/test_base_auth_manager.py
index 64d33f6065..a39b60787c 100644
--- a/tests/auth/managers/test_base_auth_manager.py
+++ b/tests/auth/managers/test_base_auth_manager.py
@@ -313,3 +313,43 @@ class TestBaseAuthManager:
assert result[1].name == "item3"
assert len(result[1].childs) == 1
assert result[1].childs[0].name == "item3.1"
+
+@patch.object(EmptyAuthManager, "security_manager")
+def test_filter_permitted_menu_items_twice(self, mock_security_manager,
auth_manager):
+mock_security_manager.has_access.side_effect = [
+# 1st call
+True, # menu 1
+False, # menu 2
+True, # menu 3
+True, # Item 3.1
+False, # Item 3.2
+# 2nd call
+False, # menu 1
+True, # menu 2
+True, # menu 3
+False, # Item 3.1
+True, # Item 3.2
+]
+
+menu = Menu()
+menu.add_link("item1")
+menu.add_link("item2")
+menu.add_link("item3")
+menu.add_link("item3.1", category="item3")
+menu.add_link("item3.2", category="item3")
+
+result = auth_manager.filter_permitted_menu_items(menu.get_list())
+
+assert len(result) == 2
+assert result[0].name == "item1"
+assert result[1].name == "item3"
+assert len(result[1].childs) == 1
+assert result[1].childs[0].name == "item3.1"
+
+result = auth_manager.filter_permitted_menu_items(menu.get_list())
+
+assert len(result) == 2
+assert result[0].name == "item2"
+assert result[1].name == "item3"
+assert len(result[1].childs) == 1
+assert result[1].childs[0].name == "item3.2"
(airflow) branch main updated (f9db9c9952 -> fece17aeed)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from f9db9c9952 Add safe-to-evict annotation to pod-template-file (#37352) add fece17aeed Run system tests using `dag.test()` (#39176) No new revisions were added by this update. Summary of changes: tests/system/conftest.py | 7 --- tests/system/utils/__init__.py | 18 +++--- 2 files changed, 7 insertions(+), 18 deletions(-)
(airflow) branch main updated: Bugfix BigQueryToMsSqlOperator (#39171)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4ae85d754e Bugfix BigQueryToMsSqlOperator (#39171)
4ae85d754e is described below
commit 4ae85d754e9f8a65d461e86eb6111d3b9974a065
Author: max <42827971+moiseen...@users.noreply.github.com>
AuthorDate: Tue Apr 23 10:47:42 2024 +
Bugfix BigQueryToMsSqlOperator (#39171)
---
.../google/cloud/transfers/bigquery_to_mssql.py| 2 +-
tests/always/test_project_structure.py | 1 -
.../cloud/transfers/test_bigquery_to_mssql.py | 88 +++
.../cloud/bigquery/example_bigquery_to_mssql.py| 276 +++--
4 files changed, 342 insertions(+), 25 deletions(-)
diff --git a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
index c251ec5615..8a5749dc9e 100644
--- a/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
+++ b/airflow/providers/google/cloud/transfers/bigquery_to_mssql.py
@@ -91,7 +91,7 @@ class BigQueryToMsSqlOperator(BigQueryToSqlBaseOperator):
self.source_project_dataset_table = source_project_dataset_table
def get_sql_hook(self) -> MsSqlHook:
-return MsSqlHook(schema=self.database,
mysql_conn_id=self.mssql_conn_id)
+return MsSqlHook(schema=self.database,
mssql_conn_id=self.mssql_conn_id)
def persist_links(self, context: Context) -> None:
project_id, dataset_id, table_id =
self.source_project_dataset_table.split(".")
diff --git a/tests/always/test_project_structure.py
b/tests/always/test_project_structure.py
index 4341dd1aec..3437092e65 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -143,7 +143,6 @@ class TestProjectStructure:
"tests/providers/google/cloud/operators/vertex_ai/test_model_service.py",
"tests/providers/google/cloud/operators/vertex_ai/test_pipeline_job.py",
"tests/providers/google/cloud/sensors/test_dataform.py",
-"tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py",
"tests/providers/google/cloud/transfers/test_bigquery_to_sql.py",
"tests/providers/google/cloud/transfers/test_mssql_to_gcs.py",
"tests/providers/google/cloud/transfers/test_presto_to_gcs.py",
diff --git a/tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py
b/tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py
new file mode 100644
index 00..e4fd897324
--- /dev/null
+++ b/tests/providers/google/cloud/transfers/test_bigquery_to_mssql.py
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from unittest import mock
+
+from airflow.providers.google.cloud.transfers.bigquery_to_mssql import
BigQueryToMsSqlOperator
+
+TASK_ID = "test-bq-create-table-operator"
+TEST_DATASET = "test-dataset"
+TEST_TABLE_ID = "test-table-id"
+TEST_DAG_ID = "test-bigquery-operators"
+TEST_PROJECT = "test-project"
+
+
+class TestBigQueryToMsSqlOperator:
+
@mock.patch("airflow.providers.google.cloud.transfers.bigquery_to_mssql.BigQueryTableLink")
+
@mock.patch("airflow.providers.google.cloud.transfers.bigquery_to_sql.BigQueryHook")
+def test_execute_good_request_to_bq(self, mock_hook, mock_link):
+destination_table = "table"
+operator = BigQueryToMsSqlOperator(
+task_id=TASK_ID,
+
source_project_dataset_table=f"{TEST_PROJECT}.{TEST_DATASET}.{TEST_TABLE_ID}",
+target_table_name=destination_table,
+replace=False,
+)
+
+operator.execute(None)
+mock_hook.return_value.list_rows.assert_called_once_with(
+dataset_id=TEST_DATASET,
+table_id=TEST_TABLE_ID,
+max_results=1000,
+selected_fields=No
(airflow) branch main updated (b82c428697 -> 4384024b07)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from b82c428697 Add warning about run/import Airflow under the Windows (#39196) add 4384024b07 Zero warnings in CLI tests (#39191) No new revisions were added by this update. Summary of changes: tests/cli/commands/test_connection_command.py | 66 +++ tests/cli/commands/test_task_command.py | 2 +- tests/deprecations_ignore.yml | 10 3 files changed, 37 insertions(+), 41 deletions(-)
(airflow) branch main updated (7bb1d4a117 -> b82c428697)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 7bb1d4a117 Use class-bound attribute directly in SA `lazyload` (#39195) add b82c428697 Add warning about run/import Airflow under the Windows (#39196) No new revisions were added by this update. Summary of changes: airflow/__init__.py | 16 1 file changed, 12 insertions(+), 4 deletions(-)
(airflow) branch main updated: Use class-bound attribute directly in SA `lazyload` (#39195)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7bb1d4a117 Use class-bound attribute directly in SA `lazyload` (#39195)
7bb1d4a117 is described below
commit 7bb1d4a11752423e63609d237092aa1152b4912b
Author: Andrey Anshin
AuthorDate: Tue Apr 23 14:01:42 2024 +0400
Use class-bound attribute directly in SA `lazyload` (#39195)
---
airflow/jobs/scheduler_job_runner.py | 2 +-
airflow/models/taskinstance.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/airflow/jobs/scheduler_job_runner.py
b/airflow/jobs/scheduler_job_runner.py
index 2d2c657dac..631de5692e 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -1631,7 +1631,7 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
query = (
select(TI)
-.options(lazyload("dag_run")) # avoids double join to
dag_run
+.options(lazyload(TI.dag_run)) # avoids double join
to dag_run
.where(TI.state.in_(State.adoptable_states))
.join(TI.queued_by_job)
.where(Job.state.is_distinct_from(JobState.RUNNING))
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index b4d5d5d65a..cb071e2edc 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -1854,7 +1854,7 @@ class TaskInstance(Base, LoggingMixin):
) -> TaskInstance | TaskInstancePydantic | None:
query = (
session.query(TaskInstance)
-.options(lazyload("dag_run")) # lazy load dag run to avoid
locking it
+.options(lazyload(TaskInstance.dag_run)) # lazy load dag run to
avoid locking it
.filter_by(
dag_id=dag_id,
run_id=run_id,
(airflow) branch main updated: Fix system tests (#39193)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e3e6776576 Fix system tests (#39193)
e3e6776576 is described below
commit e3e67765765b8c0404aa926629e826d9600d437e
Author: max <42827971+moiseen...@users.noreply.github.com>
AuthorDate: Tue Apr 23 09:59:18 2024 +
Fix system tests (#39193)
---
.../google/cloud/gcs/example_mysql_to_gcs.py | 194 ---
.../cloud/sql_to_sheets/example_sql_to_sheets.py | 277 ++---
.../cloud/transfers/example_postgres_to_gcs.py | 200 +++
3 files changed, 327 insertions(+), 344 deletions(-)
diff --git a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py
b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py
index 5b96b49c1d..884149d984 100644
--- a/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py
+++ b/tests/system/providers/google/cloud/gcs/example_mysql_to_gcs.py
@@ -62,21 +62,38 @@ PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT",
"example-project")
REGION = "europe-west2"
ZONE = REGION + "-a"
NETWORK = "default"
+CONNECTION_ID = f"mysql_{DAG_ID}_{ENV_ID}".replace("-", "_")
+CONNECTION_TYPE = "mysql"
+
+BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
+FILE_NAME = "result.json"
DB_NAME = "testdb"
DB_PORT = 3306
DB_USER_NAME = "root"
DB_USER_PASSWORD = "demo_password"
+SETUP_MYSQL_COMMAND = f"""
+sudo apt update &&
+sudo apt install -y docker.io &&
+sudo docker run -d -p {DB_PORT}:{DB_PORT} --name {DB_NAME} \
+-e MYSQL_ROOT_PASSWORD={DB_USER_PASSWORD} \
+-e MYSQL_DATABASE={DB_NAME} \
+mysql:8.1.0
+"""
+SQL_TABLE = "test_table"
+SQL_CREATE = f"CREATE TABLE IF NOT EXISTS {DB_NAME}.{SQL_TABLE} (col_1 INT,
col_2 VARCHAR(8))"
+SQL_INSERT = f"INSERT INTO {DB_NAME}.{SQL_TABLE} (col_1, col_2) VALUES (1,
'one'), (2, 'two')"
+SQL_SELECT = f"SELECT * FROM {DB_NAME}.{SQL_TABLE}"
-SHORT_MACHINE_TYPE_NAME = "n1-standard-1"
-DB_INSTANCE_NAME = f"instance-{DAG_ID}-{ENV_ID}".replace("_", "-")
+GCE_MACHINE_TYPE = "n1-standard-1"
+GCE_INSTANCE_NAME = f"instance-{DAG_ID}-{ENV_ID}".replace("_", "-")
GCE_INSTANCE_BODY = {
-"name": DB_INSTANCE_NAME,
-"machine_type": f"zones/{ZONE}/machineTypes/{SHORT_MACHINE_TYPE_NAME}",
+"name": GCE_INSTANCE_NAME,
+"machine_type": f"zones/{ZONE}/machineTypes/{GCE_MACHINE_TYPE}",
"disks": [
{
"boot": True,
-"device_name": DB_INSTANCE_NAME,
+"device_name": GCE_INSTANCE_NAME,
"initialize_params": {
"disk_size_gb": "10",
"disk_type": f"zones/{ZONE}/diskTypes/pd-balanced",
@@ -92,56 +109,41 @@ GCE_INSTANCE_BODY = {
}
],
}
-DELETE_PERSISTENT_DISK = f"""
+FIREWALL_RULE_NAME = f"allow-http-{DB_PORT}"
+CREATE_FIREWALL_RULE_COMMAND = f"""
if [ $AIRFLOW__API__GOOGLE_KEY_PATH ]; then \
gcloud auth activate-service-account
--key-file=$AIRFLOW__API__GOOGLE_KEY_PATH; \
fi;
-gcloud compute disks delete {DB_INSTANCE_NAME} --project={PROJECT_ID}
--zone={ZONE} --quiet
+if [ -z gcloud compute firewall-rules list --filter=name:{FIREWALL_RULE_NAME}
--format="value(name)" ]; then \
+gcloud compute firewall-rules create {FIREWALL_RULE_NAME} \
+ --project={PROJECT_ID} \
+ --direction=INGRESS \
+ --priority=100 \
+ --network={NETWORK} \
+ --action=ALLOW \
+ --rules=tcp:{DB_PORT} \
+ --source-ranges=0.0.0.0/0
+else
+echo "Firewall rule {FIREWALL_RULE_NAME} already exists."
+fi
"""
-
-SETUP_MYSQL = f"""
-sudo apt update &&
-sudo apt install -y docker.io &&
-sudo docker run -d -p {DB_PORT}:{DB_PORT} --name {DB_NAME} \
--e MYSQL_ROOT_PASSWORD={DB_USER_PASSWORD} \
--e MYSQL_DATABASE={DB_NAME} \
-mysql:8.1.0
-"""
-
-FIREWALL_RULE_NAME = f"allow-http-{DB_PORT}"
-CREATE_FIREWALL_RULE = f"""
+DELETE_FIREWALL_RULE_COMMAND = f"""
if [ $AIRFLOW__API__GOOGLE_KEY_PATH ]; then \
gcloud auth activate-service-account
--key-file=$AIRFLOW__API__GOOGLE_KEY_PATH; \
+fi; \
+if [ gcloud compute firewall-rules list --filter=name:{FIREWALL_RULE_NAME}
--format="value(name)" ]; then \
+gcloud compute firewall-rules delete {FIREWALL_RULE_NAME}
--project={PROJECT_ID} --quiet;
(airflow) branch main updated (e29f2d7e58 -> 63c87cd6cd)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from e29f2d7e58 Improved mocking of Context with TaskInstance for use in unit tests msgraph operator (#39157) add 63c87cd6cd Add stacklevel into the `AzureSynapsePipelineHook` deprecation warnings (#39192) No new revisions were added by this update. Summary of changes: airflow/providers/microsoft/azure/hooks/synapse.py | 1 + pyproject.toml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-)
(airflow) branch main updated: Improved mocking of Context with TaskInstance for use in unit tests msgraph operator (#39157)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e29f2d7e58 Improved mocking of Context with TaskInstance for use in
unit tests msgraph operator (#39157)
e29f2d7e58 is described below
commit e29f2d7e588234a5db3bb140b7846a1e028533f8
Author: David Blain
AuthorDate: Tue Apr 23 11:58:02 2024 +0200
Improved mocking of Context with TaskInstance for use in unit tests msgraph
operator (#39157)
-
Co-authored-by: David Blain
---
tests/providers/microsoft/azure/base.py | 38 ++---
tests/providers/microsoft/conftest.py | 28 +---
2 files changed, 22 insertions(+), 44 deletions(-)
diff --git a/tests/providers/microsoft/azure/base.py
b/tests/providers/microsoft/azure/base.py
index 4cda62858e..cad6c1449f 100644
--- a/tests/providers/microsoft/azure/base.py
+++ b/tests/providers/microsoft/azure/base.py
@@ -19,57 +19,23 @@ from __future__ import annotations
import asyncio
from contextlib import contextmanager
from copy import deepcopy
-from datetime import datetime
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any
from unittest.mock import patch
from kiota_http.httpx_request_adapter import HttpxRequestAdapter
from airflow.exceptions import TaskDeferred
-from airflow.models import Operator, TaskInstance
from airflow.providers.microsoft.azure.hooks.msgraph import
KiotaRequestAdapterHook
-from airflow.utils.session import NEW_SESSION
-from airflow.utils.xcom import XCOM_RETURN_KEY
from tests.providers.microsoft.conftest import get_airflow_connection,
mock_context
if TYPE_CHECKING:
-from sqlalchemy.orm import Session
-
+from airflow.models import Operator
from airflow.triggers.base import BaseTrigger, TriggerEvent
-class MockedTaskInstance(TaskInstance):
-values = {}
-
-def xcom_pull(
-self,
-task_ids: Iterable[str] | str | None = None,
-dag_id: str | None = None,
-key: str = XCOM_RETURN_KEY,
-include_prior_dates: bool = False,
-session: Session = NEW_SESSION,
-*,
-map_indexes: Iterable[int] | int | None = None,
-default: Any | None = None,
-) -> Any:
-self.task_id = task_ids
-self.dag_id = dag_id
-return self.values.get(f"{task_ids}_{dag_id}_{key}")
-
-def xcom_push(
-self,
-key: str,
-value: Any,
-execution_date: datetime | None = None,
-session: Session = NEW_SESSION,
-) -> None:
-self.values[f"{self.task_id}_{self.dag_id}_{key}"] = value
-
-
class Base:
def teardown_method(self, method):
KiotaRequestAdapterHook.cached_request_adapters.clear()
-MockedTaskInstance.values.clear()
@contextmanager
def patch_hook_and_request_adapter(self, response):
diff --git a/tests/providers/microsoft/conftest.py
b/tests/providers/microsoft/conftest.py
index 78d8748a89..dfba931023 100644
--- a/tests/providers/microsoft/conftest.py
+++ b/tests/providers/microsoft/conftest.py
@@ -29,6 +29,7 @@ from httpx import Response
from msgraph_core import APIVersion
from airflow.models import Connection
+from airflow.utils.context import Context
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -103,7 +104,7 @@ def mock_response(status_code, content: Any = None) ->
Response:
return response
-def mock_context(task):
+def mock_context(task) -> Context:
from datetime import datetime
from airflow.models import TaskInstance
@@ -111,9 +112,20 @@ def mock_context(task):
from airflow.utils.state import TaskInstanceState
from airflow.utils.xcom import XCOM_RETURN_KEY
+values = {}
+
class MockedTaskInstance(TaskInstance):
-def __init__(self):
-super().__init__(task=task, run_id="run_id",
state=TaskInstanceState.RUNNING)
+def __init__(
+self,
+task,
+execution_date: datetime | None = None,
+run_id: str | None = "run_id",
+state: str | None = TaskInstanceState.RUNNING,
+map_index: int = -1,
+):
+super().__init__(
+task=task, execution_date=execution_date, run_id=run_id,
state=state, map_index=map_index
+)
self.values = {}
def xcom_pull(
@@ -127,9 +139,7 @@ def mock_context(task):
map_indexes: Iterable[int] | int | None = None,
default: Any | None = None,
) -> Any:
-self.task_id = task_ids
-self.dag_id = dag_id
-return self.values.get(f"{task_ids}_{dag_id}_{key}")
+return values.get(f"{task_ids or self.task_id}_{dag_id or
self.
(airflow) branch main updated: Correct `hatch env show` command guide in airflow/INSTALL (#39194)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 32fbde1f63 Correct `hatch env show` command guide in airflow/INSTALL (#39194) 32fbde1f63 is described below commit 32fbde1f63e9940dd77885b26a51ce274d01b36c Author: Pankaj Koti AuthorDate: Tue Apr 23 14:22:08 2024 +0530 Correct `hatch env show` command guide in airflow/INSTALL (#39194) --- INSTALL | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index d237a79d04..975232bad7 100644 --- a/INSTALL +++ b/INSTALL @@ -137,7 +137,7 @@ or install all Python versions that are used in Airflow: Airflow has some pre-defined virtualenvs that you can use to develop and test airflow. You can see the list of available envs with: -hatch show env +hatch env show This is what it shows currently: @@ -154,7 +154,7 @@ This is what it shows currently: ├─┼─┼───┤ │ airflow-311 │ virtual │ Environment with Python 3.11. No devel installed │ ├─┼─┼───┤ -│ airflow-312 │ virtual │ Environment with Python 3.11. No devel installed │ +│ airflow-312 │ virtual │ Environment with Python 3.12. No devel installed │ └─┴─┴───┘ The default env (if you have not used one explicitly) is `default` and it is a Python 3.8
(airflow) branch main updated (665da50567 -> 9c1fb649c3)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 665da50567 Fixup python client release guide (#39182) add 9c1fb649c3 Executor loader don't check db compat with db isolation (#39190) No new revisions were added by this update. Summary of changes: airflow/executors/executor_loader.py | 4 1 file changed, 4 insertions(+)
(airflow) branch main updated: Fixup python client release guide (#39182)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 665da50567 Fixup python client release guide (#39182)
665da50567 is described below
commit 665da50567413485c46dca3f9fd6f6ba7e288e97
Author: Jed Cunningham <66968678+jedcunning...@users.noreply.github.com>
AuthorDate: Tue Apr 23 03:52:10 2024 -0400
Fixup python client release guide (#39182)
This also fixes 1 line in the vote email for Airflow core.
---
dev/README_RELEASE_AIRFLOW.md | 2 +-
dev/README_RELEASE_PYTHON_CLIENT.md | 61 +++--
2 files changed, 32 insertions(+), 31 deletions(-)
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 182e88d933..b265280ba7 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -401,7 +401,7 @@ are encouraged to test the release and vote with
"(non-binding)".
The test procedure for PMC members is described in:
https://github.com/apache/airflow/blob/main/dev/README_RELEASE_AIRFLOW.md#verify-the-release-candidate-by-pmc-members
-The test procedure for and Contributors who would like to test this RC is
described in:
+The test procedure for contributors and members of the community who would
like to test this RC is described in:
https://github.com/apache/airflow/blob/main/dev/README_RELEASE_AIRFLOW.md#verify-the-release-candidate-by-contributors
diff --git a/dev/README_RELEASE_PYTHON_CLIENT.md
b/dev/README_RELEASE_PYTHON_CLIENT.md
index e002661e45..1130762d9c 100644
--- a/dev/README_RELEASE_PYTHON_CLIENT.md
+++ b/dev/README_RELEASE_PYTHON_CLIENT.md
@@ -66,7 +66,7 @@ cd ..
```bash
# If you have not done so yet
-git clone g...@github.com/apache/airflow-client-python
+git clone g...@github.com:apache/airflow-client-python
cd airflow-client-python
# Checkout the right branch
git checkout main
@@ -80,8 +80,8 @@ cd ..
```bash
cd ${AIRFLOW_REPO_ROOT}
-VERSION="2.8.0"
-VERSION_SUFFIX="rc1"
+export VERSION="2.8.0"
+export VERSION_SUFFIX="rc1"
echo "${VERSION}" > clients/python/version.txt
```
@@ -101,21 +101,6 @@ git log 2.8.0..HEAD --pretty=oneline --
airflow/api_connexion/openapi/v1.yaml
- Merge it to the `v2-*-stable` branch. You will release API client from the
latest `v2-*-stable` branch
of Airflow repository - same branch that is used to release Airflow.
-- Tag your release with RC candidate tag (note that this is the RC tag even if
version of the packages
- is the same as the final version. This is because when the packages get
approved and released they
- will turn into official release and must be binary identical to the RC
packages in SVN). The tags
- should be set in both Airflow and Airflow Client repositories (with
python-client prefix in Airflow repo and
- without the prefix in the Python Client repo).
-
-```shell script
-cd ${AIRFLOW_REPO_ROOT}
-git tag -s python-client-${VERSION}${VERSION_SUFFIX} -m "Airflow Python Client
${VERSION}${VERSION_SUFFIX}"
-git push apache python-client-${VERSION}${VERSION_SUFFIX}
-cd ${CLIENT__REPO_ROOT}
-git tag -s ${VERSION}${VERSION_SUFFIX} -m "Airflow Python Client
${VERSION}${VERSION_SUFFIX}"
-git push apache ${VERSION}${VERSION_SUFFIX}
-```
-
- Build the sdist and wheel packages to be added to SVN and copy generated
client sources to the
Python Client repository.
@@ -135,11 +120,30 @@ breeze release-management prepare-python-client
--package-format both --python-c
```shell script
cd ${CLIENT_REPO_ROOT}
git diff HEAD
+git checkout -b release-${VERSION}
git add .
git commit -m "Update Python Client to ${VERSION}${VERSION_SUFFIX}"
-git push origin main
+git push apache release-${VERSION}
```
+Then open a PR and merge it into main.
+
+- Tag your release with RC candidate tag (note that this is the RC tag even if
version of the packages
+ is the same as the final version. This is because when the packages get
approved and released they
+ will turn into official release and must be binary identical to the RC
packages in SVN). The tags
+ should be set in both Airflow and Airflow Client repositories (with
python-client prefix in Airflow repo and
+ without the prefix in the Python Client repo).
+
+```shell script
+cd ${AIRFLOW_REPO_ROOT}
+git tag -s python-client-${VERSION}${VERSION_SUFFIX} -m "Airflow Python Client
${VERSION}${VERSION_SUFFIX}"
+git push apache python-client-${VERSION}${VERSION_SUFFIX}
+cd ${CLIENT_REPO_ROOT}
+git tag -s ${VERSION}${VERSION_SUFFIX} -m "Airflow Python Client
${VERSION}${VERSION_SUFFIX}"
+git push apache ${VERSION}${VERSION_SUFFIX}
+```
+
+
- Generate signatures and checksum files for the packages (if you have not
generated a key yet, generate
it by following i
(airflow) branch main updated: Only replace "client" subdir when copying sources to python client (#39181)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 57995d098e Only replace "client" subdir when copying sources to python
client (#39181)
57995d098e is described below
commit 57995d098e7051d0be6cfa6e614eb898261237f7
Author: Jarek Potiuk
AuthorDate: Tue Apr 23 00:41:11 2024 +0200
Only replace "client" subdir when copying sources to python client (#39181)
When the client is generated, only the "client" subdirectory should
be replaced rather than "airflow_client" parent so that README and
gitignores are not removed.
---
.../src/airflow_breeze/commands/release_management_commands.py | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
index ba2834375a..7dfccba03b 100644
--- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py
@@ -2702,7 +2702,11 @@ SOURCE_API_YAML_PATH = AIRFLOW_SOURCES_ROOT / "airflow"
/ "api_connexion" / "ope
TARGET_API_YAML_PATH = PYTHON_CLIENT_DIR_PATH / "v1.yaml"
OPENAPI_GENERATOR_CLI_VER = "5.4.0"
-GENERATED_CLIENT_DIRECTORIES_TO_COPY = ["airflow_client", "docs", "test"]
+GENERATED_CLIENT_DIRECTORIES_TO_COPY: list[Path] = [
+Path("airflow_client") / "client",
+Path("docs"),
+Path("test"),
+]
FILES_TO_COPY_TO_CLIENT_REPO = [
".gitignore",
".openapi-generator-ignore",
(airflow) branch main updated (ecc5af70eb -> 0d6ffeff6e)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from ecc5af70eb openlineage, snowflake: do not run external queries for Snowflake (#39113) add 0d6ffeff6e Upgrade to latest build/install dependencies (#39160) No new revisions were added by this update. Summary of changes: .pre-commit-config.yaml | 6 +++--- Dockerfile| 2 +- Dockerfile.ci | 4 ++-- clients/python/pyproject.toml | 2 +- docker_tests/requirements.txt | 2 +- pyproject.toml| 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-)
(airflow) branch main updated: Capture warnings during collect DAGs (#39109)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0d9d8fa9fa Capture warnings during collect DAGs (#39109)
0d9d8fa9fa is described below
commit 0d9d8fa9fa1368092083a581006bfc96ce57da17
Author: Andrey Anshin
AuthorDate: Mon Apr 22 13:27:11 2024 +0400
Capture warnings during collect DAGs (#39109)
* Capture warnings during collect DAGs
* Reraise captured warnings
* Update airflow/models/dagbag.py
* warnings -> warning_num in FileLoadStat
-
Co-authored-by: Tzu-ping Chung
---
airflow/models/dagbag.py | 34 ++---
airflow/utils/warnings.py| 40
tests/dags/test_dag_warnings.py | 50 +
tests/dags/test_dag_warnings.zip | Bin 0 -> 983 bytes
tests/models/test_dagbag.py | 49 +++-
tests/utils/test_warnings.py | 79 +++
6 files changed, 246 insertions(+), 6 deletions(-)
diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py
index 1902c985be..1afea71c00 100644
--- a/airflow/models/dagbag.py
+++ b/airflow/models/dagbag.py
@@ -58,6 +58,7 @@ from airflow.utils.retries import MAX_DB_RETRIES,
run_with_db_retries
from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.timeout import timeout
from airflow.utils.types import NOTSET
+from airflow.utils.warnings import capture_with_reraise
if TYPE_CHECKING:
from sqlalchemy.orm import Session
@@ -67,13 +68,23 @@ if TYPE_CHECKING:
class FileLoadStat(NamedTuple):
-"""Information about single file."""
+"""
+Information about single file.
+
+:param file: Loaded file.
+:param duration: Time spent on process file.
+:param dag_num: Total number of DAGs loaded in this file.
+:param task_num: Total number of Tasks loaded in this file.
+:param dags: DAGs names loaded in this file.
+:param warning_num: Total number of warnings captured from processing this
file.
+"""
file: str
duration: timedelta
dag_num: int
task_num: int
dags: str
+warning_num: int
class DagBag(LoggingMixin):
@@ -139,6 +150,7 @@ class DagBag(LoggingMixin):
# the file's last modified timestamp when we last read it
self.file_last_changed: dict[str, datetime] = {}
self.import_errors: dict[str, str] = {}
+self.captured_warnings: dict[str, tuple[str, ...]] = {}
self.has_logged = False
self.read_dags_from_db = read_dags_from_db
# Only used by read_dags_from_db=True
@@ -314,10 +326,21 @@ class DagBag(LoggingMixin):
# Ensure we don't pick up anything else we didn't mean to
DagContext.autoregistered_dags.clear()
-if filepath.endswith(".py") or not zipfile.is_zipfile(filepath):
-mods = self._load_modules_from_file(filepath, safe_mode)
-else:
-mods = self._load_modules_from_zip(filepath, safe_mode)
+self.captured_warnings.pop(filepath, None)
+with capture_with_reraise() as captured_warnings:
+if filepath.endswith(".py") or not zipfile.is_zipfile(filepath):
+mods = self._load_modules_from_file(filepath, safe_mode)
+else:
+mods = self._load_modules_from_zip(filepath, safe_mode)
+
+if captured_warnings:
+formatted_warnings = []
+for msg in captured_warnings:
+category = msg.category.__name__
+if (module := msg.category.__module__) != "builtins":
+category = f"{module}.{category}"
+formatted_warnings.append(f"{msg.filename}:{msg.lineno}:
{category}: {msg.message}")
+self.captured_warnings[filepath] = tuple(formatted_warnings)
found_dags = self._process_modules(filepath, mods,
file_last_changed_on_disk)
@@ -566,6 +589,7 @@ class DagBag(LoggingMixin):
dag_num=len(found_dags),
task_num=sum(len(dag.tasks) for dag in found_dags),
dags=str([dag.dag_id for dag in found_dags]),
+warning_num=len(self.captured_warnings.get(filepath,
[])),
)
)
except Exception as e:
diff --git a/airflow/utils/warnings.py b/airflow/utils/warnings.py
new file mode 100644
index 00..bcff4c06fa
--- /dev/null
+++ b/airflow/utils/warnings.py
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regardi
(airflow) branch main updated: Update logic to allow retries in AWS Batch Client hook to be effective (#38998)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fcb2bee1c6 Update logic to allow retries in AWS Batch Client hook to
be effective (#38998)
fcb2bee1c6 is described below
commit fcb2bee1c6bb8ee92d217cfb54f0dc3a0ce93b92
Author: Shahar Epstein <60007259+shah...@users.noreply.github.com>
AuthorDate: Mon Apr 22 12:20:10 2024 +0300
Update logic to allow retries in AWS Batch Client hook to be effective
(#38998)
Co-authored-by: evgenyslab
---
airflow/providers/amazon/aws/hooks/batch_client.py| 2 ++
tests/providers/amazon/aws/hooks/test_batch_client.py | 13 +++--
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/airflow/providers/amazon/aws/hooks/batch_client.py
b/airflow/providers/amazon/aws/hooks/batch_client.py
index 7ee2c29d73..b419239a16 100644
--- a/airflow/providers/amazon/aws/hooks/batch_client.py
+++ b/airflow/providers/amazon/aws/hooks/batch_client.py
@@ -397,6 +397,8 @@ class BatchClientHook(AwsBaseHook):
try:
response = self.get_conn().describe_jobs(jobs=[job_id])
return self.parse_job_description(job_id, response)
+except AirflowException as err:
+self.log.warning(err)
except botocore.exceptions.ClientError as err:
# Allow it to retry in case of exceeded quota limit of
requests to AWS API
if err.response.get("Error", {}).get("Code") !=
"TooManyRequestsException":
diff --git a/tests/providers/amazon/aws/hooks/test_batch_client.py
b/tests/providers/amazon/aws/hooks/test_batch_client.py
index 80d3e96d51..83385878b5 100644
--- a/tests/providers/amazon/aws/hooks/test_batch_client.py
+++ b/tests/providers/amazon/aws/hooks/test_batch_client.py
@@ -245,13 +245,14 @@ class TestBatchClient:
assert msg in str(ctx.value)
assert status in str(ctx.value)
-def test_check_job_success_raises_without_jobs(self):
+def test_check_job_success_raises_without_jobs(self, caplog):
self.client_mock.describe_jobs.return_value = {"jobs": []}
-with pytest.raises(AirflowException) as ctx:
-self.batch_client.check_job_success(JOB_ID)
-self.client_mock.describe_jobs.assert_called_once_with(jobs=[JOB_ID])
-msg = f"AWS Batch job ({JOB_ID}) description error"
-assert msg in str(ctx.value)
+with caplog.at_level(level=logging.WARNING):
+with pytest.raises(AirflowException):
+self.batch_client.check_job_success(JOB_ID)
+
self.client_mock.describe_jobs.assert_has_calls([mock.call(jobs=[JOB_ID])] * 3)
+msg = f"AWS Batch job ({JOB_ID}) description error"
+assert msg in caplog.messages[0]
def test_terminate_job(self):
self.client_mock.terminate_job.return_value = {}
(airflow) branch main updated (667ee1b21b -> 131a2e4e29)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 667ee1b21b Expose count of scheduled tasks in metrics (#38899) add 131a2e4e29 fix comment about the condition for emitting dagrun.first_task_scheduling_delay (#39153) No new revisions were added by this update. Summary of changes: airflow/models/dagrun.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
(airflow) branch main updated: Expose count of scheduled tasks in metrics (#38899)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 667ee1b21b Expose count of scheduled tasks in metrics (#38899)
667ee1b21b is described below
commit 667ee1b21b5615d5739d4b2b362db6a31817b705
Author: Jon Shea <1385+jons...@users.noreply.github.com>
AuthorDate: Mon Apr 22 10:54:30 2024 +0200
Expose count of scheduled tasks in metrics (#38899)
This pull request introduces a new metric that tracks the number of
tasks in the SCHEDULED state. This can help with monitoring and
debugging situations where tasks are stuck or otherwise accumulate in
the `SCHEDULED` state.
---
airflow/jobs/scheduler_job_runner.py | 3 +++
airflow/models/pool.py| 8 +++-
.../logging-monitoring/metrics.rst| 2 ++
tests/models/test_pool.py | 11 +++
4 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/airflow/jobs/scheduler_job_runner.py
b/airflow/jobs/scheduler_job_runner.py
index 2dfe2267a5..2d2c657dac 100644
--- a/airflow/jobs/scheduler_job_runner.py
+++ b/airflow/jobs/scheduler_job_runner.py
@@ -1587,11 +1587,14 @@ class SchedulerJobRunner(BaseJobRunner, LoggingMixin):
Stats.gauge(f"pool.queued_slots.{pool_name}", slot_stats["queued"])
Stats.gauge(f"pool.running_slots.{pool_name}",
slot_stats["running"])
Stats.gauge(f"pool.deferred_slots.{pool_name}",
slot_stats["deferred"])
+Stats.gauge(f"pool.scheduled_slots.{pool_name}",
slot_stats["scheduled"])
+
# Same metrics with tagging
Stats.gauge("pool.open_slots", slot_stats["open"],
tags={"pool_name": pool_name})
Stats.gauge("pool.queued_slots", slot_stats["queued"],
tags={"pool_name": pool_name})
Stats.gauge("pool.running_slots", slot_stats["running"],
tags={"pool_name": pool_name})
Stats.gauge("pool.deferred_slots", slot_stats["deferred"],
tags={"pool_name": pool_name})
+Stats.gauge("pool.scheduled_slots", slot_stats["scheduled"],
tags={"pool_name": pool_name})
@provide_session
def adopt_or_reset_orphaned_tasks(self, session: Session = NEW_SESSION) ->
int:
diff --git a/airflow/models/pool.py b/airflow/models/pool.py
index 3ca7293ffe..07b0cd64e4 100644
--- a/airflow/models/pool.py
+++ b/airflow/models/pool.py
@@ -42,6 +42,7 @@ class PoolStats(TypedDict):
deferred: int
queued: int
open: int
+scheduled: int
class Pool(Base):
@@ -178,11 +179,14 @@ class Pool(Base):
for pool_name, total_slots, include_deferred in pool_rows:
if total_slots == -1:
total_slots = float("inf") # type: ignore
-pools[pool_name] = PoolStats(total=total_slots, running=0,
queued=0, open=0, deferred=0)
+pools[pool_name] = PoolStats(
+total=total_slots, running=0, queued=0, open=0, deferred=0,
scheduled=0
+)
pool_includes_deferred[pool_name] = include_deferred
allowed_execution_states = EXECUTION_STATES | {
TaskInstanceState.DEFERRED,
+TaskInstanceState.SCHEDULED,
}
state_count_by_pool = session.execute(
select(TaskInstance.pool, TaskInstance.state,
func.sum(TaskInstance.pool_slots))
@@ -205,6 +209,8 @@ class Pool(Base):
stats_dict["queued"] = count
elif state == TaskInstanceState.DEFERRED:
stats_dict["deferred"] = count
+elif state == TaskInstanceState.SCHEDULED:
+stats_dict["scheduled"] = count
else:
raise AirflowException(f"Unexpected state. Expected values:
{allowed_execution_states}.")
diff --git
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
index 4c91f37b71..efe565094a 100644
---
a/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
+++
b/docs/apache-airflow/administration-and-deployment/logging-monitoring/metrics.rst
@@ -238,6 +238,8 @@ Name
Description
``pool.running_slots`` Number of running slots in
the pool. Metric with pool_name tagging.
``pool.deferred_slots.`` Number of deferred slots
in the pool
``pool.deferred_slots``
(airflow) branch main updated (59084fd1f4 -> aa5581e30d)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 59084fd1f4 fix: add retry logic in case of google auth refresh credential error (#38961) add aa5581e30d Update providers metadata 2024-04-21 (#39152) No new revisions were added by this update. Summary of changes: generated/provider_metadata.json | 12 1 file changed, 12 insertions(+)
(airflow) branch main updated: Cleanup PagerdutyNotifier __init__ (#39145)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 4374a49872 Cleanup PagerdutyNotifier __init__ (#39145) 4374a49872 is described below commit 4374a4987218a05913253b368f45b5fe0ef21493 Author: David Traina AuthorDate: Sat Apr 20 02:22:18 2024 -0500 Cleanup PagerdutyNotifier __init__ (#39145) fix duplicate initialization of self.class_type and self.custom_details + allow pagerduty_events_conn_id to be None --- airflow/providers/pagerduty/notifications/pagerduty.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/pagerduty/notifications/pagerduty.py b/airflow/providers/pagerduty/notifications/pagerduty.py index 5f04e2cb6c..68dae03e7c 100644 --- a/airflow/providers/pagerduty/notifications/pagerduty.py +++ b/airflow/providers/pagerduty/notifications/pagerduty.py @@ -84,7 +84,7 @@ class PagerdutyNotifier(BaseNotifier): class_type: str | None = None, images: list[Any] | None = None, links: list[Any] | None = None, -pagerduty_events_conn_id: str = "pagerduty_events_default", +pagerduty_events_conn_id: str | None = "pagerduty_events_default", integration_key: str | None = None, ): super().__init__() @@ -96,11 +96,9 @@ class PagerdutyNotifier(BaseNotifier): self.action = action self.dedup_key = dedup_key self.custom_details = custom_details -self.custom_details = custom_details self.group = group self.component = component self.class_type = class_type -self.class_type = class_type self.images = images self.links = links
(airflow) branch main updated: Resolve `B028` (no-explicit-stacklevel) in core (#39123)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 79286d6398 Resolve `B028` (no-explicit-stacklevel) in core (#39123)
79286d6398 is described below
commit 79286d6398e8e964a467de7f1e9fc10520bba00a
Author: Andrey Anshin
AuthorDate: Sat Apr 20 11:21:22 2024 +0400
Resolve `B028` (no-explicit-stacklevel) in core (#39123)
* Resolve `B028` (no-explicit-stacklevel) in core
* Update airflow/cli/commands/connection_command.py
Co-authored-by: Tzu-ping Chung
-
Co-authored-by: Tzu-ping Chung
---
airflow/__init__.py| 1 +
.../endpoints/forward_to_fab_endpoint.py | 1 +
airflow/cli/commands/connection_command.py | 12 +++---
airflow/cli/commands/dag_command.py| 2 ++
airflow/cli/commands/db_command.py | 3 ++-
airflow/configuration.py | 8 +--
airflow/decorators/task_group.py | 2 +-
airflow/jobs/scheduler_job_runner.py | 3 +++
airflow/jobs/triggerer_job_runner.py | 10 +++--
.../pre_7_4_0_compatibility/pod_generator.py | 12 +++---
airflow/logging_config.py | 1 +
airflow/metrics/otel_logger.py | 4 +++-
airflow/metrics/validators.py | 3 +--
airflow/models/baseoperator.py | 4 +++-
airflow/models/connection.py | 3 ++-
airflow/models/mappedoperator.py | 2 +-
airflow/models/param.py| 3 +++
airflow/models/xcom.py | 1 +
airflow/providers_manager.py | 1 +
airflow/serialization/serialized_objects.py| 1 +
airflow/settings.py| 1 +
airflow/utils/context.py | 7 --
airflow/utils/dot_renderer.py | 6 -
airflow/www/app.py | 1 +
airflow/www/extensions/init_views.py | 1 +
pyproject.toml | 26 +-
26 files changed, 73 insertions(+), 46 deletions(-)
diff --git a/airflow/__init__.py b/airflow/__init__.py
index 2df5571707..65bf6ef5f6 100644
--- a/airflow/__init__.py
+++ b/airflow/__init__.py
@@ -82,6 +82,7 @@ def __getattr__(name: str):
f"Python version constraint {name!r} is deprecated and will be
removed in the future. "
f"Please get version info from the 'sys.version_info'.",
DeprecationWarning,
+stacklevel=2,
)
return sys.version_info >= (3, int(py_minor))
diff --git a/airflow/api_connexion/endpoints/forward_to_fab_endpoint.py
b/airflow/api_connexion/endpoints/forward_to_fab_endpoint.py
index 6f4b417f83..9785a5b053 100644
--- a/airflow/api_connexion/endpoints/forward_to_fab_endpoint.py
+++ b/airflow/api_connexion/endpoints/forward_to_fab_endpoint.py
@@ -49,6 +49,7 @@ def _require_fab(func: Callable) -> Callable:
"This API endpoint is deprecated. "
"Please use the API under /auth/fab/v1 instead for this
operation.",
DeprecationWarning,
+stacklevel=1, # This decorator wrapped multiple times, better
point to this file
)
return func(*args, **kwargs)
diff --git a/airflow/cli/commands/connection_command.py
b/airflow/cli/commands/connection_command.py
index 2bc1076f2b..c76a648887 100644
--- a/airflow/cli/commands/connection_command.py
+++ b/airflow/cli/commands/connection_command.py
@@ -162,7 +162,9 @@ def connections_export(args):
"""Export all connections to a file."""
file_formats = [".yaml", ".json", ".env"]
if args.format:
-warnings.warn("Option `--format` is deprecated. Use `--file-format`
instead.", DeprecationWarning)
+warnings.warn(
+"Option `--format` is deprecated. Use `--file-format` instead.",
DeprecationWarning, stacklevel=3
+)
if args.format and args.file_format:
raise SystemExit("Option `--format` is deprecated. Use
`--file-format` instead.")
default_format = ".json"
@@ -223,10 +225,14 @@ def connections_add(args):
raise SystemExit("Cannot supply both conn-uri and conn-json")
if has_type and args.conn_type not in _get_connection_types():
-warnings.warn(f"The type provided to --conn-type is invalid:
{args.conn_type}")
+warnings.warn(
+f"The type provided to --conn-type is invalid: {args.conn
(airflow) branch main updated (2358064fdf -> 85bc9afcaa)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 2358064fdf Use `declarative_base` from `sqlalchemy.orm` instead of `sqlalchemy.ext.declarative` (#39134) add 85bc9afcaa fixes templated env vars for k8s pod operator (#39139) No new revisions were added by this update. Summary of changes: .../backcompat/backwards_compat_converters.py | 14 ++-- airflow/providers/cncf/kubernetes/operators/pod.py | 12 +-- pyproject.toml | 2 +- .../cncf/kubernetes/operators/test_pod.py | 41 ++ 4 files changed, 57 insertions(+), 12 deletions(-)
(airflow) branch v2-9-test updated (eee047821d -> e61cb8fa41)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from eee047821d Upgrade to latest hatchling 1.24.1 (again). add e61cb8fa41 Apply PROVIDE_PROJECT_ID mypy workaround across Google provider (#39129) No new revisions were added by this update. Summary of changes: airflow/providers/google/cloud/hooks/bigquery.py | 65 -- airflow/providers/google/cloud/hooks/cloud_sql.py | 9 ++- .../cloud/hooks/cloud_storage_transfer_service.py | 8 ++- .../providers/google/cloud/hooks/compute_ssh.py| 3 +- airflow/providers/google/cloud/hooks/dataplex.py | 8 ++- airflow/providers/google/cloud/hooks/dlp.py| 28 +- airflow/providers/google/cloud/hooks/gcs.py| 8 ++- airflow/providers/google/cloud/hooks/gdm.py| 4 +- .../google/cloud/hooks/kubernetes_engine.py| 4 +- airflow/providers/google/cloud/hooks/mlengine.py | 12 ++-- airflow/providers/google/cloud/hooks/pubsub.py | 2 +- .../providers/google/cloud/hooks/secret_manager.py | 4 +- .../providers/google/cloud/log/gcs_task_handler.py | 3 +- .../providers/google/cloud/operators/bigquery.py | 27 - .../google/cloud/operators/bigquery_dts.py | 7 ++- .../providers/google/cloud/operators/bigtable.py | 13 +++-- .../google/cloud/operators/cloud_build.py | 23 .../google/cloud/operators/cloud_memorystore.py| 33 +-- .../providers/google/cloud/operators/cloud_sql.py | 20 +++ .../operators/cloud_storage_transfer_service.py| 15 ++--- .../providers/google/cloud/operators/compute.py| 23 .../google/cloud/operators/datacatalog.py | 41 +++--- .../providers/google/cloud/operators/dataflow.py | 15 ++--- .../providers/google/cloud/operators/datafusion.py | 21 +++ .../google/cloud/operators/datapipeline.py | 5 +- .../providers/google/cloud/operators/dataprep.py | 9 +-- .../providers/google/cloud/operators/dataproc.py | 33 +-- .../providers/google/cloud/operators/datastore.py | 15 ++--- airflow/providers/google/cloud/operators/dlp.py| 61 ++-- .../providers/google/cloud/operators/functions.py | 7 ++- airflow/providers/google/cloud/operators/gcs.py| 4 +- .../google/cloud/operators/kubernetes_engine.py| 21 +++ .../google/cloud/operators/life_sciences.py| 3 +- .../providers/google/cloud/operators/mlengine.py | 21 +++ airflow/providers/google/cloud/operators/pubsub.py | 11 ++-- .../providers/google/cloud/operators/spanner.py| 13 +++-- .../google/cloud/operators/speech_to_text.py | 3 +- .../google/cloud/operators/stackdriver.py | 21 +++ airflow/providers/google/cloud/operators/tasks.py | 27 - .../google/cloud/operators/text_to_speech.py | 3 +- .../google/cloud/operators/translate_speech.py | 3 +- airflow/providers/google/cloud/operators/vision.py | 25 + .../providers/google/cloud/operators/workflows.py | 19 --- .../google/cloud/secrets/secret_manager.py | 3 +- .../providers/google/cloud/sensors/bigquery_dts.py | 3 +- airflow/providers/google/cloud/sensors/bigtable.py | 3 +- .../sensors/cloud_storage_transfer_service.py | 3 +- airflow/providers/google/cloud/sensors/dataflow.py | 9 +-- .../providers/google/cloud/sensors/datafusion.py | 3 +- airflow/providers/google/cloud/sensors/dataproc.py | 5 +- airflow/providers/google/cloud/sensors/tasks.py| 3 +- .../providers/google/cloud/sensors/workflows.py| 3 +- .../google/cloud/transfers/bigquery_to_gcs.py | 3 +- .../google/cloud/transfers/gcs_to_bigquery.py | 3 +- .../providers/google/cloud/triggers/bigquery.py| 6 +- .../providers/google/cloud/triggers/cloud_sql.py | 3 +- .../triggers/cloud_storage_transfer_service.py | 5 +- .../providers/google/cloud/triggers/dataproc.py| 3 +- .../providers/google/cloud/triggers/mlengine.py| 3 +- .../providers/google/common/hooks/base_google.py | 4 +- .../providers/google/firebase/hooks/firestore.py | 4 +- .../google/firebase/operators/firestore.py | 3 +- 62 files changed, 425 insertions(+), 347 deletions(-)
(airflow) branch main updated (fd8a05739f -> eee17f0a26)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from fd8a05739f Bugfix to correct GCSHook being called even when not required with BeamRunPythonPipelineOperator (#38716) add eee17f0a26 fix: BigQueryCheckOperator skipped value and error check in deferrable mode (#38408) No new revisions were added by this update. Summary of changes: .../providers/google/cloud/operators/bigquery.py | 30 ++--- .../google/cloud/operators/test_bigquery.py| 77 ++ 2 files changed, 86 insertions(+), 21 deletions(-)
(airflow) branch main updated: Bugfix to correct GCSHook being called even when not required with BeamRunPythonPipelineOperator (#38716)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new fd8a05739f Bugfix to correct GCSHook being called even when not
required with BeamRunPythonPipelineOperator (#38716)
fd8a05739f is described below
commit fd8a05739f945643b5023db15d51a97459109a02
Author: Zack Strathe <59071005+zstra...@users.noreply.github.com>
AuthorDate: Fri Apr 19 03:40:19 2024 -0500
Bugfix to correct GCSHook being called even when not required with
BeamRunPythonPipelineOperator (#38716)
* Bugfix to correct GCSHook being called even when not required with
BeamRunPythonPipelineOperator
* remove unneccary check for GCSHook and add unit test for
BeamRunPythonPipelineOperator to ensure that GCSHook is only called when
necessary
* Split out unit tests for TestBeamRunPythonPipelineOperator with GCSHook
'gs://' arg prefixes
* Fix formatting
---
airflow/providers/apache/beam/operators/beam.py| 3 +-
tests/providers/apache/beam/operators/test_beam.py | 73 ++
2 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/apache/beam/operators/beam.py
b/airflow/providers/apache/beam/operators/beam.py
index e88923bc05..62f650f19a 100644
--- a/airflow/providers/apache/beam/operators/beam.py
+++ b/airflow/providers/apache/beam/operators/beam.py
@@ -364,11 +364,12 @@ class
BeamRunPythonPipelineOperator(BeamBasePipelineOperator):
def execute_sync(self, context: Context):
with ExitStack() as exit_stack:
-gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id)
if self.py_file.lower().startswith("gs://"):
+gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id)
tmp_gcs_file =
exit_stack.enter_context(gcs_hook.provide_file(object_url=self.py_file))
self.py_file = tmp_gcs_file.name
if self.snake_case_pipeline_options.get("requirements_file",
"").startswith("gs://"):
+gcs_hook = GCSHook(gcp_conn_id=self.gcp_conn_id)
tmp_req_file = exit_stack.enter_context(
gcs_hook.provide_file(object_url=self.snake_case_pipeline_options["requirements_file"])
)
diff --git a/tests/providers/apache/beam/operators/test_beam.py
b/tests/providers/apache/beam/operators/test_beam.py
index f7ca9649fb..a6a4c31c77 100644
--- a/tests/providers/apache/beam/operators/test_beam.py
+++ b/tests/providers/apache/beam/operators/test_beam.py
@@ -256,6 +256,79 @@ class TestBeamRunPythonPipelineOperator:
op.on_kill()
dataflow_cancel_job.assert_not_called()
+@mock.patch(BEAM_OPERATOR_PATH.format("BeamHook"))
+@mock.patch(BEAM_OPERATOR_PATH.format("GCSHook"))
+def test_execute_gcs_hook_not_called_without_gs_prefix(self,
mock_gcs_hook, _):
+"""
+Test that execute method does not call GCSHook when neither py_file
nor requirements_file
+starts with 'gs://'. (i.e., running pipeline entirely locally)
+"""
+local_test_op_args = {
+"task_id": TASK_ID,
+"py_file": "local_file.py",
+"py_options": ["-m"],
+"default_pipeline_options": {
+"project": TEST_PROJECT,
+"requirements_file": "local_requirements.txt",
+},
+"pipeline_options": {"output": "test_local/output", "labels":
{"foo": "bar"}},
+}
+
+op = BeamRunPythonPipelineOperator(**local_test_op_args)
+context_mock = mock.MagicMock()
+
+op.execute(context_mock)
+mock_gcs_hook.assert_not_called()
+
+@mock.patch(BEAM_OPERATOR_PATH.format("BeamHook"))
+@mock.patch(BEAM_OPERATOR_PATH.format("GCSHook"))
+def test_execute_gcs_hook_called_with_gs_prefix_py_file(self,
mock_gcs_hook, _):
+"""
+Test that execute method calls GCSHook when only 'py_file' starts with
'gs://'.
+"""
+local_test_op_args = {
+"task_id": TASK_ID,
+"py_file": "gs://gcs_file.py",
+"py_options": ["-m"],
+"default_pipeline_options": {
+"project": TEST_PROJECT,
+"requirements_file": "local_requirements.txt",
+},
+"pipeline_options": {"output": "test_local/output", "labels":
{"foo": "bar"}},
+}
+op = BeamRunPythonPipeli
(airflow) branch main updated (17e60b0a2b -> 7b654b4731)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 17e60b0a2b fix: Use prefixes instead of all file paths for OpenLineage datasets in GCSDeleteObjectsOperator (#39059) add 7b654b4731 Add option to support cloudpickle in PythonVenv/External Operator (#38531) No new revisions were added by this update. Summary of changes: INSTALL| 6 +- airflow/operators/python.py| 88 +++-- .../12_airflow_dependencies_and_extras.rst | 6 +- docs/apache-airflow/extra-packages-ref.rst | 2 + docs/apache-airflow/howto/operator/python.rst | 5 + docs/spelling_wordlist.txt | 1 + hatch_build.py | 3 + pyproject.toml | 6 +- tests/decorators/test_external_python.py | 83 +++- tests/decorators/test_python_virtualenv.py | 127 +++- tests/operators/test_python.py | 217 +++-- 11 files changed, 486 insertions(+), 58 deletions(-)
(airflow) branch main updated: Fix/helm chart redis docs (#39115)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 889047094b Fix/helm chart redis docs (#39115) 889047094b is described below commit 889047094b569a51c13a0fd8d8cf6afa187eef55 Author: rom sharon <33751805+romsharo...@users.noreply.github.com> AuthorDate: Thu Apr 18 17:11:40 2024 +0300 Fix/helm chart redis docs (#39115) * add log for running callback * revert * fix docs --- docs/helm-chart/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/helm-chart/index.rst b/docs/helm-chart/index.rst index e86fa71dce..adbec48536 100644 --- a/docs/helm-chart/index.rst +++ b/docs/helm-chart/index.rst @@ -155,7 +155,7 @@ To run database migrations with Argo CD automatically, you will need to add: This will run database migrations every time there is a ``Sync`` event in Argo CD. While it is not ideal to run the migrations on every sync, it is a trade-off that allows them to be run automatically. -If you use the Celery(Kubernetes)Executor with the built-in Redis, it is recommended that you set up a static Redis password either by supplying ``redis.passwordSecretName`` and ``redis.data.brokerUrlSecretName`` or ``redis.password``. +If you use the Celery(Kubernetes)Executor with the built-in Redis, it is recommended that you set up a static Redis password either by supplying ``redis.passwordSecretName`` and ``data.brokerUrlSecretName`` or ``redis.password``. Naming Conventions
(airflow) branch main updated: tests: Add missing tests for OpenLineage (#39111)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d1ad3470cb tests: Add missing tests for OpenLineage (#39111)
d1ad3470cb is described below
commit d1ad3470cb549e23e4f11f82b1ead47eb1fac5d9
Author: Kacper Muda
AuthorDate: Thu Apr 18 14:08:17 2024 +0200
tests: Add missing tests for OpenLineage (#39111)
Signed-off-by: Kacper Muda
---
tests/always/test_project_structure.py | 3 --
...test_openlineage_adapter.py => test_adapter.py} | 0
tests/providers/openlineage/plugins/test_facets.py | 37 +++
.../openlineage/{utils => }/test_sqlparser.py | 53 +-
4 files changed, 89 insertions(+), 4 deletions(-)
diff --git a/tests/always/test_project_structure.py
b/tests/always/test_project_structure.py
index d14885aca9..4341dd1aec 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -158,9 +158,6 @@ class TestProjectStructure:
"tests/providers/microsoft/azure/operators/test_adls.py",
"tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py",
"tests/providers/mongo/sensors/test_mongo.py",
-"tests/providers/openlineage/plugins/test_adapter.py",
-"tests/providers/openlineage/plugins/test_facets.py",
-"tests/providers/openlineage/test_sqlparser.py",
"tests/providers/redis/operators/test_redis_publish.py",
"tests/providers/redis/sensors/test_redis_key.py",
"tests/providers/slack/notifications/test_slack_notifier.py",
diff --git a/tests/providers/openlineage/plugins/test_openlineage_adapter.py
b/tests/providers/openlineage/plugins/test_adapter.py
similarity index 100%
rename from tests/providers/openlineage/plugins/test_openlineage_adapter.py
rename to tests/providers/openlineage/plugins/test_adapter.py
diff --git a/tests/providers/openlineage/plugins/test_facets.py
b/tests/providers/openlineage/plugins/test_facets.py
new file mode 100644
index 00..dd4e5851f2
--- /dev/null
+++ b/tests/providers/openlineage/plugins/test_facets.py
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.providers.openlineage.plugins.facets import AirflowRunFacet
+
+
+def test_airflow_run_facet():
+dag = {"dag_id": "123"}
+dag_run = {"dag_run_id": "456"}
+task = {"task_id": "789"}
+task_instance = {"task_instance_id": "000"}
+task_uuid = "XXX"
+
+airflow_run_facet = AirflowRunFacet(
+dag=dag, dagRun=dag_run, task=task, taskInstance=task_instance,
taskUuid=task_uuid
+)
+
+assert airflow_run_facet.dag == dag
+assert airflow_run_facet.dagRun == dag_run
+assert airflow_run_facet.task == task
+assert airflow_run_facet.taskInstance == task_instance
+assert airflow_run_facet.taskUuid == task_uuid
diff --git a/tests/providers/openlineage/utils/test_sqlparser.py
b/tests/providers/openlineage/test_sqlparser.py
similarity index 87%
rename from tests/providers/openlineage/utils/test_sqlparser.py
rename to tests/providers/openlineage/test_sqlparser.py
index 24967f56b6..22bd067422 100644
--- a/tests/providers/openlineage/utils/test_sqlparser.py
+++ b/tests/providers/openlineage/test_sqlparser.py
@@ -32,7 +32,7 @@ from openlineage.client.run import Dataset
from openlineage.common.sql import DbTableMeta
from airflow.providers.openlineage.extractors import OperatorLineage
-from airflow.providers.openlineage.sqlparser import DatabaseInfo, SQLParser
+from airflow.providers.openlineage.sqlparser import DatabaseInfo,
GetTableSchemasParams, SQLParser
DB_NAME = "FOOD_DELIVERY"
DB_SCHEMA_NAME = "PUBLIC"
@@ -41,6 +41,57 @@ DB_TABLE_NAME = DbTableMeta("DISCOUNTS")
NAMESPACE = "test_namespace&q
(airflow) branch main updated: Switch committer builds to public runners (#39106)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ab9444e0c2 Switch committer builds to public runners (#39106)
ab9444e0c2 is described below
commit ab9444e0c26df1f2acfb83bb47bb64acefec437a
Author: Jarek Potiuk
AuthorDate: Thu Apr 18 12:14:43 2024 +0200
Switch committer builds to public runners (#39106)
Following the discussion on devlist, we are switching committer buids to
public runners.
Discussion:
* https://lists.apache.org/thread/f2kxdjm12nf2rh0tqxmv3rj49sjd6c3w
---
.../src/airflow_breeze/utils/selective_checks.py | 5 +
dev/breeze/tests/test_selective_checks.py| 20 ++--
2 files changed, 11 insertions(+), 14 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index ee12f2881c..6b42d91346 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1090,10 +1090,7 @@ class SelectiveChecks:
if USE_SELF_HOSTED_RUNNERS_LABEL in self._pr_labels:
# Forced self-hosted runners
return RUNS_ON_SELF_HOSTED_RUNNER
-if actor in COMMITTERS:
-return RUNS_ON_SELF_HOSTED_RUNNER
-else:
-return RUNS_ON_PUBLIC_RUNNER
+return RUNS_ON_PUBLIC_RUNNER
return RUNS_ON_PUBLIC_RUNNER
@cached_property
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 099478ea9c..e2ee1f0a6b 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1700,13 +1700,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
(),
dict(),
-'["self-hosted", "Linux", "X64"]',
-"true",
-"true",
-"true",
+'["ubuntu-22.04"]',
+"false",
"false",
"true",
"false",
+"false",
+"false",
False,
id="Pull request committer",
),
@@ -1780,13 +1780,13 @@ def test_helm_tests_trigger_ci_build(files: tuple[str,
...], expected_outputs: d
"apache/airflow",
[],
dict(),
-'["self-hosted", "Linux", "X64"]',
-"true",
-"true",
-"true",
+'["ubuntu-22.04"]',
+"false",
"false",
"true",
"false",
+"false",
+"false",
False,
id="Pull request target committer",
),
@@ -2065,7 +2065,7 @@ def test_mypy_matches(
("README.md",),
{
"is-committer-build": "true",
-"runs-on-as-json-default": '["self-hosted", "Linux", "X64"]',
+"runs-on-as-json-default": '["ubuntu-22.04"]',
},
"potiuk",
(),
@@ -2075,7 +2075,7 @@ def test_mypy_matches(
("README.md",),
{
"is-committer-build": "false",
-"runs-on-as-json-default": '["self-hosted", "Linux", "X64"]',
+"runs-on-as-json-default": '["ubuntu-22.04"]',
},
"potiuk",
("non committer build",),
(airflow) branch v2-9-test updated (a4027b7bf8 -> eee047821d)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from a4027b7bf8 update simple-salesforce type hints to support 1.12.6 (#39047) add 789051c41a Fix main failing because of the new xmlsec 1.3.14 compatibility (#39104) add eee047821d Upgrade to latest hatchling 1.24.1 (again). No new revisions were added by this update. Summary of changes: .pre-commit-config.yaml| 6 +++--- airflow/providers/amazon/provider.yaml | 4 clients/python/pyproject.toml | 2 +- docker_tests/requirements.txt | 2 +- generated/provider_dependencies.json | 3 ++- pyproject.toml | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-)
(airflow) branch main updated (b41cf629c8 -> cd888870ba)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from b41cf629c8 Deferrable mode for Dataflow sensors (#37693) add cd70ba Fix main failing because of the new xmlsec 1.3.14 compatibility (#39104) No new revisions were added by this update. Summary of changes: airflow/providers/amazon/provider.yaml | 4 generated/provider_dependencies.json | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-)
(airflow) branch v2-9-test updated (dbebce4c4e -> a4027b7bf8)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from dbebce4c4e Update hatchling to version 1.24.0 (#39072) add a4027b7bf8 update simple-salesforce type hints to support 1.12.6 (#39047) No new revisions were added by this update. Summary of changes: airflow/providers/salesforce/hooks/salesforce.py | 6 -- airflow/providers/salesforce/operators/bulk.py | 16 +--- 2 files changed, 13 insertions(+), 9 deletions(-)
(airflow) branch main updated: update document (#39068)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new b56c7cd335 update document (#39068) b56c7cd335 is described below commit b56c7cd3352bf8f493e081bda895058320122b51 Author: humit AuthorDate: Wed Apr 17 05:46:14 2024 +0900 update document (#39068) --- docs/apache-airflow/extra-packages-ref.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst index 91922dd5fc..0a9a73600d 100644 --- a/docs/apache-airflow/extra-packages-ref.rst +++ b/docs/apache-airflow/extra-packages-ref.rst @@ -57,8 +57,6 @@ python dependencies for the provided package. +-+-++ | google-auth | ``pip install 'apache-airflow[google-auth]'`` | Google auth backend| +-+-++ -| graphviz| ``pip install 'apache-airflow[graphvis]'`` | Enables exporting DAGs to .dot graphical output| -+-+-++ | graphviz| ``pip install 'apache-airflow[graphviz]'`` | Graphviz renderer for converting DAG to graphical output | +-+-++ | kerberos| ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized services (Hadoop, Presto, Trino) | @@ -333,7 +331,7 @@ pre-installed when Airflow is installed. +-+-+--+--+ | openlineage | ``pip install 'apache-airflow[openlineage]'`` | Sending OpenLineage events | | +-+-+--+--+ -| opensearch | ``pip install 'apache-airflow[opensearch]'`` | Opensearch hooks and operators | | +| opensearch | ``pip install 'apache-airflow[opensearch]'``| Opensearch hooks and operators | | +-+-+--+--+ | papermill | ``pip install 'apache-airflow[papermill]'`` | Papermill hooks and operators| | +-+-+--+--+
(airflow) branch main updated: Moves airflow import in deprecated pod_generator to local (#39062)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new b59cef1c83 Moves airflow import in deprecated pod_generator to local (#39062) b59cef1c83 is described below commit b59cef1c8374d53e1ffdeb46c94e6c9ff30c239d Author: Jarek Potiuk AuthorDate: Tue Apr 16 15:20:53 2024 +0200 Moves airflow import in deprecated pod_generator to local (#39062) The import might be invoked when K8S executor starts with sentry on and it might lead to circular imports Related: #31442 --- airflow/providers/cncf/kubernetes/pod_generator_deprecated.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py b/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py index 92baf41e68..9a978cbd08 100644 --- a/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py +++ b/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py @@ -31,8 +31,6 @@ import uuid import re2 from kubernetes.client import models as k8s -from airflow.utils.hashlib_wrapper import md5 - MAX_POD_ID_LEN = 253 MAX_LABEL_LEN = 63 @@ -71,6 +69,8 @@ def make_safe_label_value(string): way from the original value sent to this function, then we need to truncate to 53 chars, and append it with a unique hash. """ +from airflow.utils.hashlib_wrapper import md5 + safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string) if len(safe_label) > MAX_LABEL_LEN or string != safe_label:
(airflow) branch main updated (b00274156b -> d82246a0a7)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from b00274156b [docs] update `DagBag` class docstring to include all params (#38814) add d82246a0a7 Data aware scheduling docs edits (#38687) No new revisions were added by this update. Summary of changes: .../authoring-and-scheduling/datasets.rst | 65 +++--- 1 file changed, 32 insertions(+), 33 deletions(-)
(airflow) branch main updated: [docs] update `DagBag` class docstring to include all params (#38814)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new b00274156b [docs] update `DagBag` class docstring to include all params (#38814) b00274156b is described below commit b00274156b28cce6a68dac20e667d21d00310016 Author: Kalyan AuthorDate: Tue Apr 16 17:52:57 2024 +0530 [docs] update `DagBag` class docstring to include all params (#38814) * update docstring for DagBag class * break long line * fix space Signed-off-by: kalyanr - Signed-off-by: kalyanr --- airflow/models/dagbag.py | 5 + 1 file changed, 5 insertions(+) diff --git a/airflow/models/dagbag.py b/airflow/models/dagbag.py index ce9bf5587b..1902c985be 100644 --- a/airflow/models/dagbag.py +++ b/airflow/models/dagbag.py @@ -89,11 +89,16 @@ class DagBag(LoggingMixin): :param dag_folder: the folder to scan to find DAGs :param include_examples: whether to include the examples that ship with airflow or not +:param safe_mode: when ``False``, scans all python modules for dags. +When ``True`` uses heuristics (files containing ``DAG`` and ``airflow`` strings) +to filter python modules to scan for dags. :param read_dags_from_db: Read DAGs from DB if ``True`` is passed. If ``False`` DAGs are read from python files. +:param store_serialized_dags: deprecated parameter, same effect as `read_dags_from_db` :param load_op_links: Should the extra operator link be loaded via plugins when de-serializing the DAG? This flag is set to False in Scheduler so that Extra Operator links are not loaded to not run User code in Scheduler. +:param collect_dags: when True, collects dags during class initialization. """ def __init__(
(airflow) branch main updated (4d94df489a -> 6995eafc8a)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 4d94df489a Add changelog for airflow python client 2.9.0 (#39060) add 6995eafc8a Upgrade to latest hatchling as build dependency (#39044) No new revisions were added by this update. Summary of changes: .pre-commit-config.yaml | 6 +++--- clients/python/pyproject.toml | 2 +- docker_tests/requirements.txt | 2 +- pyproject.toml| 2 +- 4 files changed, 6 insertions(+), 6 deletions(-)
(airflow) branch main updated (82e51760a1 -> 1e9d3d1c35)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 82e51760a1 Hostname was "redacted", not "redact"; remove it when there is no context (#39037) add 1e9d3d1c35 Do not add local airflow sources in CI when building k8s image (#39043) No new revisions were added by this update. Summary of changes: .github/workflows/k8s-tests.yml| 2 +- .../doc/images/output_k8s_build-k8s-image.svg | 62 +--- .../doc/images/output_k8s_build-k8s-image.txt | 2 +- .../doc/images/output_k8s_run-complete-tests.svg | 86 -- .../doc/images/output_k8s_run-complete-tests.txt | 2 +- .../airflow_breeze/commands/kubernetes_commands.py | 25 ++- .../commands/kubernetes_commands_config.py | 2 + 7 files changed, 111 insertions(+), 70 deletions(-)
(airflow) branch main updated: Add four unit tests for aws/utils (#38820)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 10d5654c9b Add four unit tests for aws/utils (#38820)
10d5654c9b is described below
commit 10d5654c9b0ce46a74de60abaab2b067a2a77482
Author: Steven Blake <9101623+slycyber...@users.noreply.github.com>
AuthorDate: Mon Apr 15 10:13:01 2024 -0700
Add four unit tests for aws/utils (#38820)
---
tests/always/test_project_structure.py| 2 -
tests/providers/amazon/aws/utils/test_sqs.py | 137 ++
tests/providers/amazon/aws/utils/test_tags.py | 50 ++
3 files changed, 187 insertions(+), 2 deletions(-)
diff --git a/tests/always/test_project_structure.py
b/tests/always/test_project_structure.py
index 1b965ec998..cd8e68594c 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -82,8 +82,6 @@ class TestProjectStructure:
"tests/providers/amazon/aws/triggers/test_step_function.py",
"tests/providers/amazon/aws/utils/test_rds.py",
"tests/providers/amazon/aws/utils/test_sagemaker.py",
-"tests/providers/amazon/aws/utils/test_sqs.py",
-"tests/providers/amazon/aws/utils/test_tags.py",
"tests/providers/amazon/aws/waiters/test_base_waiter.py",
"tests/providers/apache/cassandra/hooks/test_cassandra.py",
"tests/providers/apache/drill/operators/test_drill.py",
diff --git a/tests/providers/amazon/aws/utils/test_sqs.py
b/tests/providers/amazon/aws/utils/test_sqs.py
new file mode 100644
index 00..74f9c309fb
--- /dev/null
+++ b/tests/providers/amazon/aws/utils/test_sqs.py
@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import jsonpath_ng
+import pytest
+
+from airflow.providers.amazon.aws.utils.sqs import (
+filter_messages,
+filter_messages_jsonpath,
+filter_messages_literal,
+process_response,
+)
+
+
+@pytest.fixture
+def response_json():
+return {"Messages": [{"Body": "message1"}, {"Body": "message2"}]}
+
+
+@pytest.fixture
+def response_nested_dict():
+return [{"Body": '{"key": "value1"}'}, {"Body": '{"key": "value2"}'}]
+
+
+@pytest.fixture
+def response_dict():
+return [{"Body": "message1"}, {"Body": "message2"}, {"Body": "message3"}]
+
+
+def test_process_response_with_empty_response():
+response_json = {}
+processed_response = process_response(response_json)
+assert processed_response == []
+
+
+def test_process_response_with_no_messages(response_json):
+response_json["Messages"] = []
+processed_response = process_response(response_json)
+assert processed_response == []
+
+
+def test_process_response_with_messages_and_no_filtering(response_json):
+processed_response = process_response(response_json)
+assert processed_response == response_json["Messages"]
+
+
+def test_process_response_with_messages_and_literal_filtering(response_json):
+processed_response = process_response(
+response_json, message_filtering="literal",
message_filtering_match_values=["message1"]
+)
+assert processed_response == [{"Body": "message1"}]
+
+
+def test_filter_messages_literal():
+messages = [{"Body": "message1"}, {"Body": "message2"}]
+filtered_messages = filter_messages(
+messages,
+message_filtering="literal",
+message_filtering_match_values=["message1"],
+message_filtering_config="",
+)
+assert filtered_messages == [{"Body": "message1"}]
+
+
+def test_filter_messages_jsonpath(response_nested_dict):
+filtered_messages = filter_messages(
+re
(airflow) branch main updated (1ded297509 -> 0c630fadad)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 1ded297509 Remove flag from AWS auth manager to use it (#39033) add 0c630fadad handle KubernetesDeleteJobOperator import (#39036) No new revisions were added by this update. Summary of changes: .../providers/google/cloud/operators/kubernetes_engine.py| 12 +++- 1 file changed, 11 insertions(+), 1 deletion(-)
(airflow) branch main updated (f902dbe91f -> 246f697873)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from f902dbe91f Ban `unittest` marks which presented into the `pytest` (#39025) add 246f697873 Fix PROD image package installation in CI (#39035) No new revisions were added by this update. Summary of changes: Dockerfile| 4 dev/breeze/src/airflow_breeze/params/build_prod_params.py | 13 + docker_tests/test_prod_image.py | 6 ++ scripts/docker/install_from_docker_context_files.sh | 4 4 files changed, 27 insertions(+)
(airflow) branch fix-prod-image-ci-preparation updated (34da29f58a -> 7ddc9bc2ab)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch fix-prod-image-ci-preparation in repository https://gitbox.apache.org/repos/asf/airflow.git discard 34da29f58a Fix PROD image package installation in CI add 7ddc9bc2ab Fix PROD image package installation in CI This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (34da29f58a) \ N -- N -- N refs/heads/fix-prod-image-ci-preparation (7ddc9bc2ab) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: docker_tests/test_prod_image.py | 6 ++ 1 file changed, 6 insertions(+)
(airflow) 01/01: Fix PROD image package installation in CI
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch fix-prod-image-ci-preparation
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 34da29f58a1043875602fe07dee25b27ba26c9e0
Author: Jarek Potiuk
AuthorDate: Mon Apr 15 16:08:48 2024 +0200
Fix PROD image package installation in CI
When PROD image packages are installed in in CI, the local sources
should not be present in the image, also constraints from sources
shoudl replace the one downloaded from main.
---
Dockerfile| 4
dev/breeze/src/airflow_breeze/params/build_prod_params.py | 13 +
scripts/docker/install_from_docker_context_files.sh | 4
3 files changed, 21 insertions(+)
diff --git a/Dockerfile b/Dockerfile
index 94798fbdb7..9ae60c52bd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -780,6 +780,10 @@ function
install_airflow_and_providers_from_docker_context_files(){
${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint
"${local_constraints_file}" \
"${install_airflow_package[@]}"
"${installing_providers_packages[@]}"
set +x
+echo
+echo "${COLOR_BLUE}Copying ${local_constraints_file} to
${HOME}/constraints.txt${COLOR_RESET}"
+echo
+cp "${local_constraints_file}" "${HOME}/constraints.txt"
else
echo
echo "${COLOR_BLUE}Installing docker-context-files packages with
constraints from GitHub${COLOR_RESET}"
diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py
b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
index d3a3dbfdbc..d6193963ce 100644
--- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py
+++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py
@@ -143,6 +143,19 @@ class BuildProdParams(CommonBuildParams):
)
self.airflow_constraints_location = constraints_location
extra_build_flags.extend(self.args_for_remote_install)
+elif self.install_packages_from_context:
+extra_build_flags.extend(
+[
+"--build-arg",
+"AIRFLOW_SOURCES_FROM=/empty",
+"--build-arg",
+"AIRFLOW_SOURCES_TO=/empty",
+"--build-arg",
+f"AIRFLOW_INSTALLATION_METHOD={self.installation_method}",
+"--build-arg",
+
f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}",
+],
+)
else:
extra_build_flags.extend(
[
diff --git a/scripts/docker/install_from_docker_context_files.sh
b/scripts/docker/install_from_docker_context_files.sh
index d6fab1e827..edcb50c82e 100644
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -86,6 +86,10 @@ function
install_airflow_and_providers_from_docker_context_files(){
${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint
"${local_constraints_file}" \
"${install_airflow_package[@]}"
"${installing_providers_packages[@]}"
set +x
+echo
+echo "${COLOR_BLUE}Copying ${local_constraints_file} to
${HOME}/constraints.txt${COLOR_RESET}"
+echo
+cp "${local_constraints_file}" "${HOME}/constraints.txt"
else
echo
echo "${COLOR_BLUE}Installing docker-context-files packages with
constraints from GitHub${COLOR_RESET}"
(airflow) branch fix-prod-image-ci-preparation created (now 34da29f58a)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch fix-prod-image-ci-preparation in repository https://gitbox.apache.org/repos/asf/airflow.git at 34da29f58a Fix PROD image package installation in CI This branch includes the following new commits: new 34da29f58a Fix PROD image package installation in CI The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(airflow) branch main updated (21f08ddd93 -> ad1e473489)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 21f08ddd93 Check whether `AUTH_ROLE_PUBLIC` is set in
`check_authentication` (#39012)
add ad1e473489 Add a generic system test in `tests/system/` and run it in
the CI (#38967)
No new revisions were added by this update.
Summary of changes:
.github/workflows/run-unit-tests.yml | 2 ++
.github/workflows/special-tests.yml| 21 +
.../src/airflow_breeze/params/shell_params.py | 1 +
tests/conftest.py | 2 +-
tests/system/conftest.py | 7 ++
.../example_influxdb_query.py => example_empty.py} | 26 +++---
6 files changed, 45 insertions(+), 14 deletions(-)
copy tests/system/{providers/influxdb/example_influxdb_query.py =>
example_empty.py} (70%)
(airflow) branch main updated (081637e083 -> 1f0f9079c7)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 081637e083 Provide a UUID to trace log when RPC internal server error... WDYT? (#38989) add 1f0f9079c7 Ensure we can serialize DatasetEventAccessor(s) properly (#38993) No new revisions were added by this update. Summary of changes: airflow/serialization/enums.py | 2 ++ airflow/serialization/serialized_objects.py| 20 ++-- airflow/utils/context.pyi | 1 + docs/spelling_wordlist.txt | 1 + tests/serialization/test_serialized_objects.py | 12 5 files changed, 34 insertions(+), 2 deletions(-)
(airflow) branch main updated: Provide a UUID to trace log when RPC internal server error... WDYT? (#38989)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 081637e083 Provide a UUID to trace log when RPC internal server
error... WDYT? (#38989)
081637e083 is described below
commit 081637e083da02e5500550db682a869f25324096
Author: Daniel Standish <15932138+dstand...@users.noreply.github.com>
AuthorDate: Mon Apr 15 02:07:54 2024 -0700
Provide a UUID to trace log when RPC internal server error... WDYT? (#38989)
* Provide a UUID to trace log when RPC internal server error
For security reasons, we don't present the user with tracebacks when
there's a webserver error. If we similarly don't want to provide tracebacks in
task execution logs, we could provide a UUID that an admin can use to find the
error in the server logs.
(cherry picked from commit 5b6ef96989ff53f77713d00be85637d932d7d928)
* improve language of error message
---
airflow/api_internal/endpoints/rpc_api_endpoint.py | 13 +
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/airflow/api_internal/endpoints/rpc_api_endpoint.py
b/airflow/api_internal/endpoints/rpc_api_endpoint.py
index b5fd545066..f2fa53d3ee 100644
--- a/airflow/api_internal/endpoints/rpc_api_endpoint.py
+++ b/airflow/api_internal/endpoints/rpc_api_endpoint.py
@@ -21,6 +21,7 @@ import functools
import json
import logging
from typing import TYPE_CHECKING, Any, Callable
+from uuid import uuid4
from flask import Response
@@ -139,7 +140,11 @@ def internal_airflow_api(body: dict[str, Any]) ->
APIResponse:
output_json = BaseSerialization.serialize(output,
use_pydantic_models=True)
response = json.dumps(output_json) if output_json is not None else
None
return Response(response=response, headers={"Content-Type":
"application/json"})
-except Exception as e:
-log.error("Error executing method: %s.", method_name)
-log.exception(e)
-return Response(response=f"Error executing method: {method_name}.",
status=500)
+except Exception:
+error_id = uuid4()
+log.exception("Error executing method '%s'; error_id=%s.",
method_name, error_id)
+return Response(
+response=f"Error executing method '{method_name}'. "
+f"The server side traceback may be identified with
error_id={error_id}",
+status=500,
+)
(airflow) branch main updated (f8104325b7 -> bfd347759d)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from f8104325b7 Activate RUF019 that checks for unnecessary key check (#38950) add bfd347759d Correcting an example taskflow example (#39015) No new revisions were added by this update. Summary of changes: docs/apache-airflow/tutorial/taskflow.rst | 1 + 1 file changed, 1 insertion(+)
(airflow-site) branch main updated: Add Power BI plugin Github link in ecosystem (#1006)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow-site.git The following commit(s) were added to refs/heads/main by this push: new 9303c309c8 Add Power BI plugin Github link in ecosystem (#1006) 9303c309c8 is described below commit 9303c309c81c4a38dc2266c28de64d9e2621e195 Author: ambika-garg <70703123+ambika-g...@users.noreply.github.com> AuthorDate: Sun Apr 14 15:42:43 2024 -0400 Add Power BI plugin Github link in ecosystem (#1006) --- landing-pages/site/content/en/ecosystem/_index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/landing-pages/site/content/en/ecosystem/_index.md b/landing-pages/site/content/en/ecosystem/_index.md index f87c23f09d..6b6943eac3 100644 --- a/landing-pages/site/content/en/ecosystem/_index.md +++ b/landing-pages/site/content/en/ecosystem/_index.md @@ -177,6 +177,8 @@ Apache Airflow releases the [Official Apache Airflow Community Chart](https://ai [Panda Patrol](https://github.com/aivanzhang/panda_patrol#-panda-patrol) - Test and profile your data right within your Airflow DAGs. With dashboards and alerts already pre-built. +[PowerBI-Airflow-Plugin](https://github.com/ambika-garg/PowerBI_Airflow_Plugin) - The Airflow plugin for Power BI includes a custom Airflow operator designed to refresh Power BI datasets. + [Pylint-Airflow](https://github.com/BasPH/pylint-airflow) - A Pylint plugin for static code analysis on Airflow code. [Redactics](https://www.redactics.com) - A managed appliance (built on Airflow) installed next to your databases that powers a growing collection of data management workflows.
(airflow) branch main updated: Adding MSGraphOperator in Microsoft Azure provider (#38111)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 1c9a6609f3 Adding MSGraphOperator in Microsoft Azure provider (#38111) 1c9a6609f3 is described below commit 1c9a6609f36a6fabddfd6d3858cca049d4088668 Author: David Blain AuthorDate: Sun Apr 14 21:39:56 2024 +0200 Adding MSGraphOperator in Microsoft Azure provider (#38111) * refactor: Initial commit contains the new MSGraphOperator * refactor: Extracted common method into Base class for patching airflow connection and request adapter + make multiple patches into one context manager Python 3.8 compatible * refactor: Refactored some typing issues related to msgraph * refactor: Added some docstrings and fixed additional typing issues * refactor: Fixed more static checks * refactor: Added license on top of test serializer and fixed import * Revert "refactor: Added license on top of test serializer and fixed import" This reverts commit 04d6b85494b0b9d3973564d4ac5abb718ac32cc7. * refactor: Added license on top of serializer files and fixed additional static checks * refactor: Added new line at end of json test files * refactor: Try fixing docstrings on operator and serializer * refactor: Replaced NoneType with None * refactor: Made type unions Python 3.8 compatible * refactor: Reformatted some files to comply with static checks formatting * refactor: Reformatted base to comply with static checks formatting * refactor: Added msgraph-core dependency to provider.yaml * refactor: Added msgraph integration info to provider.yaml * refactor: Added init in resources * fix: Fixed typing of response_handler * refactor: Added assertions on conn_id, tenant_id, client_id and client_secret * refactor: Fixed some static checks * Revert "refactor: Added assertions on conn_id, tenant_id, client_id and client_secret" This reverts commit 88aa7dccd95b98585872ae9eb5cd339162a06bb9. * refactor: Changed imports in hook as we don't use mockito anymore we don't need the module before constructor * refactor: Renamed test methods * refactor: Replace List type with list * refactor: Moved docstring as one line * refactor: Fixed typing for tests and added test for response_handler * refactor: Refactored tests * fix: Fixed MS Graph logo filename * refactor: Fixed additional static checks remarks * refactor: Added white line in type checking block * refactor: Added msgraph-core dependency to provider_dependencies.json * refactor: Updated docstring on response handler * refactor: Moved ResponseHandler and Serializer to triggers module * docs: Added documentation on how to use the MSGraphAsyncOperator * docs: Fixed END tag in examples * refactor: Removed docstring from CallableResponseHandler * refactor: Ignore UP031 Use format specifiers instead of percent format as this is not possible here the way the DAG is evaluated in Airflow (due to XCom's) * Revert "refactor: Removed docstring from CallableResponseHandler" This reverts commit 6a14ebe01936ca31ab188ab0fcbb40ba1960c3ba. * refactor: Simplified docstring on CallableResponseHandler * refactor: Updated provider.yaml to add reference of msgraph to how-to-guide * refactor: Updated docstrings on operator and trigger * refactor: Fixed additional static checks * refactor: Ignore UP031 Use format specifiers instead of percent format as this is not possible here the way the DAG is evaluated in Airflow (due to XCom's) * refactor: Added param to docstring ResponseHandler * refactor: Updated pyproject.toml as main * refactor: Reformatted docstrings in trigger * refactor: Removed unused serialization module * fix: Fixed execution of consecutive tasks in execute_operator method * refactor: Added customizable pagination_function parameter to Operator and made operator PowerBI compatible * refactor: Reformatted operator and trigger * refactor: Added check if query_parameters is not None * refactor: Removed typing of top and odata_count * refactor: Ignore type for tenant_id (this is an issue in the ClientSecretCredential class) * refactor: Changed docstring on MSGraphTrigger * refactor: Changed docstring on MSGraphTrigger * refactor: Added docstring to handle_response_async method * refactor: Fixed docstring to imperative for handle_response_async method * refactor: Try quotin
(airflow) branch main updated: Trigger FAB provider tests on API change (#39010)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new ac1f74402f Trigger FAB provider tests on API change (#39010)
ac1f74402f is described below
commit ac1f74402f553efc61fe015bca4450d21a61e16b
Author: Jarek Potiuk
AuthorDate: Sun Apr 14 19:21:07 2024 +0200
Trigger FAB provider tests on API change (#39010)
Follow up after #38924 which was not triggered when API changed
---
.../src/airflow_breeze/utils/selective_checks.py | 134 ++---
dev/breeze/tests/test_selective_checks.py | 20 +--
2 files changed, 76 insertions(+), 78 deletions(-)
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 15919f6d66..ee12f2881c 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -304,60 +304,6 @@ def find_provider_affected(changed_file: str,
include_docs: bool) -> str | None:
return "Providers"
-def find_all_providers_affected(
-changed_files: tuple[str, ...], include_docs: bool,
fail_if_suspended_providers_affected: bool
-) -> list[str] | str | None:
-all_providers: set[str] = set()
-
-all_providers_affected = False
-suspended_providers: set[str] = set()
-for changed_file in changed_files:
-provider = find_provider_affected(changed_file,
include_docs=include_docs)
-if provider == "Providers":
-all_providers_affected = True
-elif provider is not None:
-if provider not in DEPENDENCIES:
-suspended_providers.add(provider)
-else:
-all_providers.add(provider)
-if all_providers_affected:
-return "ALL_PROVIDERS"
-if suspended_providers:
-# We check for suspended providers only after we have checked if all
providers are affected.
-# No matter if we found that we are modifying a suspended provider
individually, if all providers are
-# affected, then it means that we are ok to proceed because likely we
are running some kind of
-# global refactoring that affects multiple providers including the
suspended one. This is a
-# potential escape hatch if someone would like to modify suspended
provider,
-# but it can be found at the review time and is anyway harmless as the
provider will not be
-# released nor tested nor used in CI anyway.
-get_console().print("[yellow]You are modifying suspended providers.\n")
-get_console().print(
-"[info]Some providers modified by this change have been suspended,
"
-"and before attempting such changes you should fix the reason for
suspension."
-)
-get_console().print(
-"[info]When fixing it, you should set suspended = false in
provider.yaml "
-"to make changes to the provider."
-)
-get_console().print(f"Suspended providers: {suspended_providers}")
-if fail_if_suspended_providers_affected:
-get_console().print(
-"[error]This PR did not have `allow suspended provider
changes` label set so it will fail."
-)
-sys.exit(1)
-else:
-get_console().print(
-"[info]This PR had `allow suspended provider changes` label
set so it will continue"
-)
-if not all_providers:
-return None
-for provider in list(all_providers):
-all_providers.update(
-get_related_providers(provider, upstream_dependencies=True,
downstream_dependencies=True)
-)
-return sorted(all_providers)
-
-
def _match_files_with_regexps(files: tuple[str, ...], matched_files,
matching_regexps):
for file in files:
if any(re.match(regexp, file) for regexp in matching_regexps):
@@ -747,7 +693,7 @@ class SelectiveChecks:
# prepare all providers packages and build all providers documentation
return "Providers" in self._get_test_types_to_run()
-def _fail_if_suspended_providers_affected(self):
+def _fail_if_suspended_providers_affected(self) -> bool:
return "allow suspended provider changes" not in self._pr_labels
def _get_test_types_to_run(self) -> list[str]:
@@ -800,14 +746,17 @@ class SelectiveChecks:
get_console().print(remaining_files)
candidate_test_types.update(all_selective_test_types())
else:
-if "Providers" in candidate_test_types:
-affected_providers = find_all_providers_affected(
-changed_files=self._files,
+
(airflow) branch main updated: Revert "check whether AUTH_ROLE_PUBLIC is set in check_authentication (#38924)" (#39009)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7fc2169a41 Revert "check whether AUTH_ROLE_PUBLIC is set in
check_authentication (#38924)" (#39009)
7fc2169a41 is described below
commit 7fc2169a4156428d0f59847d3ef4ee53cb934dac
Author: Jarek Potiuk
AuthorDate: Sun Apr 14 17:24:41 2024 +0200
Revert "check whether AUTH_ROLE_PUBLIC is set in check_authentication
(#38924)" (#39009)
This reverts commit 7b608250468740954c6b0af7a5f7f23dfa52b473.
---
airflow/api_connexion/security.py | 6 -
tests/api_connexion/conftest.py| 15 +-
.../endpoints/test_config_endpoint.py | 22 ---
.../endpoints/test_connection_endpoint.py | 89 --
tests/api_connexion/endpoints/test_dag_endpoint.py | 99 ---
.../endpoints/test_dag_run_endpoint.py | 185
.../endpoints/test_dag_source_endpoint.py | 16 --
.../endpoints/test_dag_warning_endpoint.py | 12 --
.../endpoints/test_dataset_endpoint.py | 186 +
.../endpoints/test_event_log_endpoint.py | 44 -
10 files changed, 2 insertions(+), 672 deletions(-)
diff --git a/airflow/api_connexion/security.py
b/airflow/api_connexion/security.py
index 660bc6cce2..1cc044d9dd 100644
--- a/airflow/api_connexion/security.py
+++ b/airflow/api_connexion/security.py
@@ -49,12 +49,6 @@ def check_authentication() -> None:
response = auth.requires_authentication(Response)()
if response.status_code == 200:
return
-
-# Even if the current_user is anonymous, the AUTH_ROLE_PUBLIC might still
have permission.
-appbuilder = get_airflow_app().appbuilder
-if appbuilder.get_app.config.get("AUTH_ROLE_PUBLIC", None):
-return
-
# since this handler only checks authentication, not authorization,
# we should always return 401
raise Unauthenticated(headers=response.headers)
diff --git a/tests/api_connexion/conftest.py b/tests/api_connexion/conftest.py
index 481f07fe73..c860a78f27 100644
--- a/tests/api_connexion/conftest.py
+++ b/tests/api_connexion/conftest.py
@@ -40,9 +40,7 @@ def minimal_app_for_api():
)
def factory():
with conf_vars({("api", "auth_backends"):
"tests.test_utils.remote_user_api_auth_backend"}):
-_app = app.create_app(testing=True, config={"WTF_CSRF_ENABLED":
False}) # type:ignore
-_app.config["AUTH_ROLE_PUBLIC"] = None
-return _app
+return app.create_app(testing=True, config={"WTF_CSRF_ENABLED":
False}) # type:ignore
return factory()
@@ -69,14 +67,3 @@ def dagbag():
)
DagBag(include_examples=True, read_dags_from_db=False).sync_to_db()
return DagBag(include_examples=True, read_dags_from_db=True)
-
-
-@pytest.fixture
-def set_auto_role_public(request):
-app = request.getfixturevalue("minimal_app_for_api")
-auto_role_public = app.config["AUTH_ROLE_PUBLIC"]
-app.config["AUTH_ROLE_PUBLIC"] = request.param
-
-yield
-
-app.config["AUTH_ROLE_PUBLIC"] = auto_role_public
diff --git a/tests/api_connexion/endpoints/test_config_endpoint.py
b/tests/api_connexion/endpoints/test_config_endpoint.py
index 3dd5814e5d..c091c4ef1c 100644
--- a/tests/api_connexion/endpoints/test_config_endpoint.py
+++ b/tests/api_connexion/endpoints/test_config_endpoint.py
@@ -222,16 +222,6 @@ class TestGetConfig:
assert response.status_code == 403
assert "chose not to expose" in response.json["detail"]
-@pytest.mark.parametrize(
-"set_auto_role_public, expected_status_code",
-(("Public", 403), ("Admin", 200)),
-indirect=["set_auto_role_public"],
-)
-def test_with_auth_role_public_set(self, set_auto_role_public,
expected_status_code):
-response = self.client.get("/api/v1/config", headers={"Accept":
"application/json"})
-
-assert response.status_code == expected_status_code
-
class TestGetValue:
@pytest.fixture(autouse=True)
@@ -349,15 +339,3 @@ class TestGetValue:
)
assert response.status_code == 403
assert "chose not to expose" in response.json["detail"]
-
-@pytest.mark.parametrize(
-"set_auto_role_public, expected_status_code",
-(("Public", 403), ("Admin", 200)),
-indirect=["set_auto_role_public"],
-)
-def test_with_auth_role_public_set(self, set_auto_role_public,
expected_status_code):
-response = self.client.get(
(airflow) branch main updated: Partially enable `B028` rule: no-explicit-stacklevel (#39000)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new aa73bca2b5 Partially enable `B028` rule: no-explicit-stacklevel (#39000) aa73bca2b5 is described below commit aa73bca2b549b4c063d684d389e444b07ffaa888 Author: Andrey Anshin AuthorDate: Sun Apr 14 17:21:19 2024 +0400 Partially enable `B028` rule: no-explicit-stacklevel (#39000) --- pyproject.toml | 32 +++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 84fc37ee39..1bc89bbea8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -290,6 +290,7 @@ extend-select = [ "B006", # Checks for uses of mutable objects as function argument defaults. "B017", # Checks for pytest.raises context managers that catch Exception or BaseException. "B019", # Use of functools.lru_cache or functools.cache on methods can lead to memory leaks +"B028", # No explicit stacklevel keyword argument found "TRY002", # Prohibit use of `raise Exception`, use specific exceptions instead. "RUF006", # Checks for asyncio dangling task ] @@ -322,7 +323,7 @@ required-imports = ["from __future__ import annotations"] combine-as-imports = true [tool.ruff.lint.per-file-ignores] -"airflow/__init__.py" = ["F401"] +"airflow/__init__.py" = ["F401", "B028"] "airflow/models/__init__.py" = ["F401", "TCH004"] "airflow/models/sqla_models.py" = ["F401"] @@ -381,6 +382,35 @@ combine-as-imports = true "tests/providers/qdrant/operators/test_qdrant.py" = ["E402"] "tests/providers/snowflake/operators/test_snowflake_sql.py" = ["E402"] +# All the modules which do not follow B028 yet: https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/ +"airflow/api_connexion/endpoints/forward_to_fab_endpoint.py" = ["B028"] +"airflow/cli/commands/connection_command.py" = ["B028"] +"airflow/cli/commands/dag_command.py" = ["B028"] +"airflow/cli/commands/db_command.py" = ["B028"] +"airflow/configuration.py" = ["B028"] +"airflow/decorators/task_group.py" = ["B028"] +"airflow/jobs/scheduler_job_runner.py" = ["B028"] +"airflow/jobs/triggerer_job_runner.py" = ["B028"] +"airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py" = ["B028"] +"airflow/logging_config.py" = ["B028"] +"airflow/metrics/otel_logger.py" = ["B028"] +"airflow/metrics/validators.py" = ["B028"] +"airflow/models/baseoperator.py" = ["B028"] +"airflow/models/connection.py" = ["B028"] +"airflow/models/mappedoperator.py" = ["B028"] +"airflow/models/param.py" = ["B028"] +"airflow/models/xcom.py" = ["B028"] +"airflow/providers_manager.py" = ["B028"] +"airflow/serialization/serialized_objects.py" = ["B028"] +"airflow/settings.py" = ["B028"] +"airflow/utils/context.py" = ["B028"] +"airflow/utils/dot_renderer.py" = ["B028"] +"airflow/www/app.py" = ["B028"] +"airflow/www/extensions/init_views.py" = ["B028"] +"airflow/providers/jdbc/hooks/jdbc.py" = ["B028"] +"airflow/providers/microsoft/azure/hooks/synapse.py" = ["B028"] +"helm_tests/airflow_aux/test_basic_helm_chart.py" = ["B028"] + [tool.ruff.lint.flake8-tidy-imports] # Disallow all relative imports. ban-relative-imports = "all"
(airflow) branch main updated: Add `inherit_cache` attribute for `CreateTableAs` custom SA Clause (#38985)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new bdb8f749ee Add `inherit_cache` attribute for `CreateTableAs` custom SA Clause (#38985) bdb8f749ee is described below commit bdb8f749ee462d508bc9da43b16d18a599ff0e7b Author: Andrey Anshin AuthorDate: Sun Apr 14 17:19:17 2024 +0400 Add `inherit_cache` attribute for `CreateTableAs` custom SA Clause (#38985) --- airflow/utils/db_cleanup.py | 2 ++ pyproject.toml | 1 + 2 files changed, 3 insertions(+) diff --git a/airflow/utils/db_cleanup.py b/airflow/utils/db_cleanup.py index 4475209b77..7fe158e605 100644 --- a/airflow/utils/db_cleanup.py +++ b/airflow/utils/db_cleanup.py @@ -218,6 +218,8 @@ def _subquery_keep_last(*, recency_column, keep_last_filters, group_by_columns, class CreateTableAs(Executable, ClauseElement): """Custom sqlalchemy clause element for CTAS operations.""" +inherit_cache = False + def __init__(self, name, query): self.name = name self.query = query diff --git a/pyproject.toml b/pyproject.toml index f853b4e3e1..84fc37ee39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -470,6 +470,7 @@ filterwarnings = [ # Avoid building cartesian product which might impact performance "error:SELECT statement has a cartesian product between FROM:sqlalchemy.exc.SAWarning:airflow", 'error:Coercing Subquery object into a select\(\) for use in IN\(\):sqlalchemy.exc.SAWarning:airflow', +'error:Class.*will not make use of SQL compilation caching', "ignore::DeprecationWarning:flask_appbuilder.filemanager", "ignore::DeprecationWarning:flask_appbuilder.widgets", # FAB do not support SQLAclhemy 2
(airflow) branch main updated: Use `model_validate` instead of `parse_obj` for de-serialize Pydantic V2 model (#38999)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 30f56020f7 Use `model_validate` instead of `parse_obj` for
de-serialize Pydantic V2 model (#38999)
30f56020f7 is described below
commit 30f56020f74e88743eec7b225c9ac632cc356c41
Author: Andrey Anshin
AuthorDate: Sun Apr 14 17:16:34 2024 +0400
Use `model_validate` instead of `parse_obj` for de-serialize Pydantic V2
model (#38999)
---
airflow/serialization/serialized_objects.py| 12
tests/serialization/test_serialized_objects.py | 40 ++
2 files changed, 28 insertions(+), 24 deletions(-)
diff --git a/airflow/serialization/serialized_objects.py
b/airflow/serialization/serialized_objects.py
index 9cc1d41931..8dd7465c5f 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -702,17 +702,17 @@ class BaseSerialization:
return Connection(**var)
elif use_pydantic_models and _ENABLE_AIP_44:
if type_ == DAT.BASE_JOB:
-return JobPydantic.parse_obj(var)
+return JobPydantic.model_validate(var)
elif type_ == DAT.TASK_INSTANCE:
-return TaskInstancePydantic.parse_obj(var)
+return TaskInstancePydantic.model_validate(var)
elif type_ == DAT.DAG_RUN:
-return DagRunPydantic.parse_obj(var)
+return DagRunPydantic.model_validate(var)
elif type_ == DAT.DAG_MODEL:
-return DagModelPydantic.parse_obj(var)
+return DagModelPydantic.model_validate(var)
elif type_ == DAT.DATA_SET:
-return DatasetPydantic.parse_obj(var)
+return DatasetPydantic.model_validate(var)
elif type_ == DAT.LOG_TEMPLATE:
-return LogTemplatePydantic.parse_obj(var)
+return LogTemplatePydantic.model_validate(var)
elif type_ == DAT.ARG_NOT_SET:
return NOTSET
else:
diff --git a/tests/serialization/test_serialized_objects.py
b/tests/serialization/test_serialized_objects.py
index 1491a02dc2..bfeff47627 100644
--- a/tests/serialization/test_serialized_objects.py
+++ b/tests/serialization/test_serialized_objects.py
@@ -19,6 +19,7 @@ from __future__ import annotations
import inspect
import json
+import warnings
from datetime import datetime, timedelta
from importlib import import_module
@@ -311,28 +312,31 @@ sample_objects = {
)
def test_serialize_deserialize_pydantic(input, pydantic_class, encoded_type,
cmp_func):
"""If use_pydantic_models=True the objects should be serialized to
Pydantic objects."""
-pytest.importorskip("pydantic", minversion="2.0.0")
+pydantic = pytest.importorskip("pydantic", minversion="2.0.0")
from airflow.serialization.serialized_objects import BaseSerialization
-serialized = BaseSerialization.serialize(input, use_pydantic_models=True)
# does not raise
-# Verify the result is JSON-serializable
-json.dumps(serialized) # does not raise
-assert serialized["__type"] == encoded_type
-assert serialized["__var"] is not None
-deserialized = BaseSerialization.deserialize(serialized,
use_pydantic_models=True)
-assert isinstance(deserialized, pydantic_class)
-assert cmp_func(input, deserialized)
-
-# verify that when we round trip a pydantic model we get the same thing
-reserialized = BaseSerialization.serialize(deserialized,
use_pydantic_models=True)
-dereserialized = BaseSerialization.deserialize(reserialized,
use_pydantic_models=True)
-assert isinstance(dereserialized, pydantic_class)
-assert dereserialized == deserialized
+with warnings.catch_warnings():
+warnings.simplefilter("error",
category=pydantic.warnings.PydanticDeprecationWarning)
-# Verify recursive behavior
-obj = [[input]]
-BaseSerialization.serialize(obj, use_pydantic_models=True) # does not
raise
+serialized = BaseSerialization.serialize(input,
use_pydantic_models=True) # does not raise
+# Verify the result is JSON-serializable
+json.dumps(serialized) # does not raise
+assert serialized["__type"] == encoded_type
+assert serialized["__var"] is not None
+deserialized = BaseSerialization.deserialize(serialized,
use_pydantic_models=True)
+assert isinstance(deserialized, pydantic_class)
+assert cmp_func(input, deserialized)
+
+# verify that when we round trip a pydantic model we get the same thing
+reserialized = BaseSerialization.serialize(deserialized,
use_pydantic_models=True)
+dereserialized = BaseSeri
(airflow) branch main updated (70e17f6e00 -> 710d28f341)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 70e17f6e00 Mark `extending/add-build-essential-extend/Dockerfile` docker example test as XFAIL (#38978) add 710d28f341 Update testing_packages.rst (#38996) No new revisions were added by this update. Summary of changes: contributing-docs/testing/testing_packages.rst | 41 +- 1 file changed, 21 insertions(+), 20 deletions(-)
(airflow) branch main updated: Mark `extending/add-build-essential-extend/Dockerfile` docker example test as XFAIL (#38978)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 70e17f6e00 Mark `extending/add-build-essential-extend/Dockerfile`
docker example test as XFAIL (#38978)
70e17f6e00 is described below
commit 70e17f6e0010e369a4b2d05869c66ff902c3ba42
Author: Andrey Anshin
AuthorDate: Sun Apr 14 17:11:36 2024 +0400
Mark `extending/add-build-essential-extend/Dockerfile` docker example test
as XFAIL (#38978)
---
.github/workflows/additional-prod-image-tests.yml | 2 +-
.../test_examples_of_prod_image_building.py| 28 ++
2 files changed, 25 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index b7cfc0067b..a584be8cf1 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -93,7 +93,7 @@ jobs:
test-examples-of-prod-image-building:
timeout-minutes: 60
-name: "Test examples of POD image building"
+name: "Test examples of PROD image building"
runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
diff --git a/docker_tests/test_examples_of_prod_image_building.py
b/docker_tests/test_examples_of_prod_image_building.py
index 6931a78b79..2e99b69280 100644
--- a/docker_tests/test_examples_of_prod_image_building.py
+++ b/docker_tests/test_examples_of_prod_image_building.py
@@ -52,10 +52,30 @@ def test_shell_script_example(script_file):
run_command(["bash", script_file])
-@pytest.mark.parametrize("dockerfile",
glob.glob(f"{DOCKER_EXAMPLES_DIR}/**/Dockerfile", recursive=True))
-def test_dockerfile_example(dockerfile, tmp_path):
-rel_dockerfile_path = Path(dockerfile).relative_to(DOCKER_EXAMPLES_DIR)
-image_name = str(rel_dockerfile_path).lower().replace("/", "-")
+def docker_examples(directory: Path, xfails: list[str] | None = None):
+xfails = xfails or []
+result = []
+for filepath in sorted(directory.rglob("**/Dockerfile")):
+markers = []
+rel_path = filepath.relative_to(directory).as_posix()
+if rel_path in xfails:
+markers.append(pytest.mark.xfail)
+result.append(pytest.param(filepath, rel_path, marks=markers,
id=rel_path))
+return result
+
+
+@pytest.mark.parametrize(
+"dockerfile, relative_path",
+docker_examples(
+DOCKER_EXAMPLES_DIR,
+xfails=[
+# FIXME https://github.com/apache/airflow/issues/38988
+"extending/add-build-essential-extend/Dockerfile",
+],
+),
+)
+def test_dockerfile_example(dockerfile, relative_path, tmp_path):
+image_name = relative_path.lower().replace("/", "-")
content = Path(dockerfile).read_text()
test_image = os.environ.get("TEST_IMAGE", get_latest_airflow_image())
(airflow) branch main updated (c450cbfbf9 -> 311f2dd3ff)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from c450cbfbf9 Make default_action_log an RPC function (#38946) add 311f2dd3ff Remove unnecessary logs from test output (#38974) No new revisions were added by this update. Summary of changes: tests/conftest.py | 11 +++ tests/models/test_mappedoperator.py | 4 2 files changed, 11 insertions(+), 4 deletions(-)
(airflow) branch main updated (61501796ca -> 259bc1c7b8)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 61501796ca Fix `sql_alchemy_engine_args` config example (#38971) add 259bc1c7b8 Fix Mypy error introduced in main (#38975) No new revisions were added by this update. Summary of changes: airflow/models/taskinstance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(airflow) branch main updated (5f6f4a5f46 -> 61501796ca)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 5f6f4a5f46 Fix calendar view with no DAG Run. (#38964) add 61501796ca Fix `sql_alchemy_engine_args` config example (#38971) No new revisions were added by this update. Summary of changes: airflow/config_templates/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(airflow) branch main updated (8507ad9e54 -> db8471cc07)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 8507ad9e54 Mark WASB blob tests as DB tests (#38963)
add db8471cc07 Make handling of connection by fs/adls.py closer to that of
WasbHook and add unit tests. (#38747)
No new revisions were added by this update.
Summary of changes:
airflow/providers/microsoft/azure/fs/adls.py | 62 ++-
airflow/providers/microsoft/azure/hooks/wasb.py| 34 +-
airflow/providers/microsoft/azure/utils.py | 34 ++
tests/always/test_project_structure.py | 1 -
.../providers/microsoft/azure/fs}/__init__.py | 0
tests/providers/microsoft/azure/fs/test_adls.py| 124 +
tests/providers/microsoft/azure/test_utils.py | 28 +
7 files changed, 245 insertions(+), 38 deletions(-)
copy {airflow/api/auth => tests/providers/microsoft/azure/fs}/__init__.py
(100%)
create mode 100644 tests/providers/microsoft/azure/fs/test_adls.py
(airflow) branch main updated: Mark WASB blob tests as DB tests (#38963)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8507ad9e54 Mark WASB blob tests as DB tests (#38963)
8507ad9e54 is described below
commit 8507ad9e543d7e838d7f16e8e9e8dfd3931b8f33
Author: Jarek Potiuk
AuthorDate: Fri Apr 12 14:24:34 2024 +0200
Mark WASB blob tests as DB tests (#38963)
The WASB blob tests started to fail intermittently after retries were
implemented in #38910. Without understanding all the details, it
seems that the async tests run in xdist mode of tests suffer from the
bug of pytest-xdist - because the thread that runs some of the tests
is not main and it likely interferes with signals handling and is
likely an incarnation of
https://github.com/pytest-dev/pytest-xdist/issues/620
(which still needs to be solved in pytest-xdist)
Since those two tests that keeps on failing intermittently and the other
tests have been marked as db_test - likely for the same reason, we
should simply mark all the module as db_test to avoid the interference.
---
tests/providers/microsoft/azure/triggers/test_wasb.py | 6 ++
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/tests/providers/microsoft/azure/triggers/test_wasb.py
b/tests/providers/microsoft/azure/triggers/test_wasb.py
index 1d6a185d48..9f1de42623 100644
--- a/tests/providers/microsoft/azure/triggers/test_wasb.py
+++ b/tests/providers/microsoft/azure/triggers/test_wasb.py
@@ -35,6 +35,8 @@ TEST_DATA_STORAGE_BLOB_PREFIX =
TEST_DATA_STORAGE_BLOB_NAME[:10]
TEST_WASB_CONN_ID = "wasb_default"
POKE_INTERVAL = 5.0
+pytestmark = pytest.mark.db_test
+
class TestWasbBlobSensorTrigger:
TRIGGER = WasbBlobSensorTrigger(
@@ -78,7 +80,6 @@ class TestWasbBlobSensorTrigger:
assert task.done() is False
asyncio.get_event_loop().stop()
-@pytest.mark.db_test
@pytest.mark.asyncio
@mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbAsyncHook.check_for_blob_async")
async def test_success(self, mock_check_for_blob):
@@ -115,7 +116,6 @@ class TestWasbBlobSensorTrigger:
assert message in caplog.text
asyncio.get_event_loop().stop()
-@pytest.mark.db_test
@pytest.mark.asyncio
@mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbAsyncHook.check_for_blob_async")
async def test_trigger_exception(self, mock_check_for_blob):
@@ -170,7 +170,6 @@ class TestWasbPrefixSensorTrigger:
assert task.done() is False
asyncio.get_event_loop().stop()
-@pytest.mark.db_test
@pytest.mark.asyncio
@mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbAsyncHook.check_for_prefix_async")
async def test_success(self, mock_check_for_prefix):
@@ -208,7 +207,6 @@ class TestWasbPrefixSensorTrigger:
mock_log_info.assert_called_once_with(message)
asyncio.get_event_loop().stop()
-@pytest.mark.db_test
@pytest.mark.asyncio
@mock.patch("airflow.providers.microsoft.azure.hooks.wasb.WasbAsyncHook.check_for_prefix_async")
async def test_trigger_exception(self, mock_check_for_prefix):
(airflow) branch main updated: Create GKESuspendJobOperator and GKEResumeJobOperator operators (#38677)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new c4a106e69b Create GKESuspendJobOperator and GKEResumeJobOperator operators (#38677) c4a106e69b is described below commit c4a106e69bbc396d2527a3b8c94e2d95fced4284 Author: Maksim AuthorDate: Fri Apr 12 11:51:03 2024 +0200 Create GKESuspendJobOperator and GKEResumeJobOperator operators (#38677) --- .../google/cloud/operators/kubernetes_engine.py| 211 +++- .../operators/cloud/kubernetes_engine.rst | 30 +++ .../cloud/operators/test_kubernetes_engine.py | 219 + .../example_kubernetes_engine_job.py | 26 +++ 4 files changed, 485 insertions(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/operators/kubernetes_engine.py b/airflow/providers/google/cloud/operators/kubernetes_engine.py index 8729335c5a..4b4261ff8d 100644 --- a/airflow/providers/google/cloud/operators/kubernetes_engine.py +++ b/airflow/providers/google/cloud/operators/kubernetes_engine.py @@ -29,7 +29,7 @@ import yaml from deprecated import deprecated from google.api_core.exceptions import AlreadyExists from google.cloud.container_v1.types import Cluster -from kubernetes.client import V1JobList +from kubernetes.client import V1JobList, models as k8s from kubernetes.utils.create_from_yaml import FailToCreateError from packaging.version import parse as parse_version @@ -47,6 +47,7 @@ from airflow.providers.google.cloud.hooks.kubernetes_engine import ( GKEDeploymentHook, GKEHook, GKEJobHook, +GKEKubernetesHook, GKEPodHook, ) from airflow.providers.google.cloud.links.kubernetes_engine import ( @@ -1494,3 +1495,211 @@ class GKEDeleteJobOperator(KubernetesDeleteJobOperator): ).fetch_cluster_info() return super().execute(context) + + +class GKESuspendJobOperator(GoogleCloudBaseOperator): +""" +Suspend Job by given name. + +.. seealso:: +For more information on how to use this operator, take a look at the guide: +:ref:`howto/operator:GKESuspendJobOperator` + +:param name: The name of the Job to suspend +:param project_id: The Google Developers Console project id. +:param location: The name of the Google Kubernetes Engine zone or region in which the cluster +resides. +:param cluster_name: The name of the Google Kubernetes Engine cluster. +:param namespace: The name of the Google Kubernetes Engine namespace. +:param use_internal_ip: Use the internal IP address as the endpoint. +:param gcp_conn_id: The connection ID to use connecting to Google Cloud. +:param impersonation_chain: Optional service account to impersonate using short-term +credentials, or chained list of accounts required to get the access_token +of the last account in the list, which will be impersonated in the request. +If set as a string, the account must grant the originating account +the Service Account Token Creator IAM role. +If set as a sequence, the identities from the list must grant +Service Account Token Creator IAM role to the directly preceding identity, with first +account from the list granting this role to the originating account (templated). +""" + +template_fields: Sequence[str] = ( +"project_id", +"gcp_conn_id", +"name", +"namespace", +"cluster_name", +"location", +"impersonation_chain", +) +operator_extra_links = (KubernetesEngineJobLink(),) + +def __init__( +self, +*, +name: str, +location: str, +namespace: str, +cluster_name: str, +project_id: str | None = None, +use_internal_ip: bool = False, +gcp_conn_id: str = "google_cloud_default", +impersonation_chain: str | Sequence[str] | None = None, +**kwargs, +) -> None: +super().__init__(**kwargs) + +self.project_id = project_id +self.gcp_conn_id = gcp_conn_id +self.location = location +self.name = name +self.namespace = namespace +self.cluster_name = cluster_name +self.use_internal_ip = use_internal_ip +self.impersonation_chain = impersonation_chain + +self.job: V1Job | None = None +self._ssl_ca_cert: str +self._cluster_url: str + +@cached_property +def cluster_hook(self) -> GKEHook: +return GKEHook( +gcp_conn_id=self.gcp_conn_id, +location=self.location, +impersonation_chain=self.impersonation_chain, +) + +@cached_property +def hook(self) -> GKEKubernetesH
(airflow) branch main updated: Activate RUF006 rule to detect dangling asyncio tasks (#38947)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b3b90794cb Activate RUF006 rule to detect dangling asyncio tasks
(#38947)
b3b90794cb is described below
commit b3b90794cb01df968fcb89032b8f95577a7592aa
Author: Hussein Awala
AuthorDate: Fri Apr 12 11:47:42 2024 +0200
Activate RUF006 rule to detect dangling asyncio tasks (#38947)
---
.pre-commit-config.yaml| 4 ++--
pyproject.toml | 1 +
tests/providers/google/cloud/triggers/test_bigquery.py | 2 +-
3 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 85c35c95e3..4fb1555f5f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -335,7 +335,7 @@ repos:
types_or: [python, pyi]
args: [--fix]
require_serial: true
-additional_dependencies: ["ruff==0.3.5"]
+additional_dependencies: ["ruff==0.3.6"]
exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py
- id: ruff-format
name: Run 'ruff format' for extremely fast Python formatting
@@ -345,7 +345,7 @@ repos:
types_or: [python, pyi]
args: []
require_serial: true
-additional_dependencies: ["ruff==0.3.5"]
+additional_dependencies: ["ruff==0.3.6"]
exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^airflow/contrib/
- id: replace-bad-characters
name: Replace bad characters
diff --git a/pyproject.toml b/pyproject.toml
index ee55d35bc3..0e5dcef675 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -290,6 +290,7 @@ extend-select = [
"B017", # Checks for pytest.raises context managers that catch Exception
or BaseException.
"B019", # Use of functools.lru_cache or functools.cache on methods can
lead to memory leaks
"TRY002", # Prohibit use of `raise Exception`, use specific exceptions
instead.
+"RUF006", # Checks for asyncio dangling task
]
ignore = [
"D203",
diff --git a/tests/providers/google/cloud/triggers/test_bigquery.py
b/tests/providers/google/cloud/triggers/test_bigquery.py
index ed4861ca76..8c4318fde4 100644
--- a/tests/providers/google/cloud/triggers/test_bigquery.py
+++ b/tests/providers/google/cloud/triggers/test_bigquery.py
@@ -601,7 +601,7 @@ class TestBigQueryValueCheckTrigger:
get_job_output.return_value = {}
get_records.return_value = [[2], [4]]
-asyncio.create_task(value_check_trigger.run().__anext__())
+await value_check_trigger.run().__anext__()
await asyncio.sleep(0.5)
generator = value_check_trigger.run()
(airflow) branch main updated: Suppress jaydebeapi.Error when setAutoCommit or getAutoCommit is unsupported by JDBC driver (#38707)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 41869d3840 Suppress jaydebeapi.Error when setAutoCommit or
getAutoCommit is unsupported by JDBC driver (#38707)
41869d3840 is described below
commit 41869d3840a959a8ad87e0973656606f080d8608
Author: David Blain
AuthorDate: Fri Apr 12 11:44:27 2024 +0200
Suppress jaydebeapi.Error when setAutoCommit or getAutoCommit is
unsupported by JDBC driver (#38707)
-
Co-authored-by: David Blain
---
airflow/providers/jdbc/hooks/jdbc.py| 19 +--
tests/providers/jdbc/hooks/test_jdbc.py | 28 +++-
2 files changed, 44 insertions(+), 3 deletions(-)
diff --git a/airflow/providers/jdbc/hooks/jdbc.py
b/airflow/providers/jdbc/hooks/jdbc.py
index dd592652ae..832c16b9aa 100644
--- a/airflow/providers/jdbc/hooks/jdbc.py
+++ b/airflow/providers/jdbc/hooks/jdbc.py
@@ -17,6 +17,9 @@
# under the License.
from __future__ import annotations
+import traceback
+import warnings
+from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import jaydebeapi
@@ -27,6 +30,15 @@ if TYPE_CHECKING:
from airflow.models.connection import Connection
+@contextmanager
+def suppress_and_warn(*exceptions: type[BaseException]):
+"""Context manager that suppresses the given exceptions and logs a warning
message."""
+try:
+yield
+except exceptions as e:
+warnings.warn(f"Exception suppressed: {e}\n{traceback.format_exc()}",
category=UserWarning)
+
+
class JdbcHook(DbApiHook):
"""General hook for JDBC access.
@@ -152,7 +164,8 @@ class JdbcHook(DbApiHook):
:param conn: The connection.
:param autocommit: The connection's autocommit setting.
"""
-conn.jconn.setAutoCommit(autocommit)
+with suppress_and_warn(jaydebeapi.Error):
+conn.jconn.setAutoCommit(autocommit)
def get_autocommit(self, conn: jaydebeapi.Connection) -> bool:
"""Get autocommit setting for the provided connection.
@@ -162,4 +175,6 @@ class JdbcHook(DbApiHook):
to True on the connection. False if it is either not set, set to
False, or the connection does not support auto-commit.
"""
-return conn.jconn.getAutoCommit()
+with suppress_and_warn(jaydebeapi.Error):
+return conn.jconn.getAutoCommit()
+return False
diff --git a/tests/providers/jdbc/hooks/test_jdbc.py
b/tests/providers/jdbc/hooks/test_jdbc.py
index 8b303db67e..80eedf8ee4 100644
--- a/tests/providers/jdbc/hooks/test_jdbc.py
+++ b/tests/providers/jdbc/hooks/test_jdbc.py
@@ -22,11 +22,14 @@ import logging
from unittest import mock
from unittest.mock import Mock, patch
+import jaydebeapi
import pytest
+from airflow.exceptions import DeserializingResultError
from airflow.models import Connection
-from airflow.providers.jdbc.hooks.jdbc import JdbcHook
+from airflow.providers.jdbc.hooks.jdbc import JdbcHook, suppress_and_warn
from airflow.utils import db
+from airflow.utils.context import AirflowContextDeprecationWarning
pytestmark = pytest.mark.db_test
@@ -82,6 +85,13 @@ class TestJdbcHook:
jdbc_hook.set_autocommit(jdbc_conn, False)
jdbc_conn.jconn.setAutoCommit.assert_called_once_with(False)
+@patch("airflow.providers.jdbc.hooks.jdbc.jaydebeapi.connect")
+def test_jdbc_conn_set_autocommit_when_not_supported(self, _):
+jdbc_hook = JdbcHook()
+jdbc_conn = jdbc_hook.get_conn()
+jdbc_conn.jconn.setAutoCommit.side_effect = jaydebeapi.Error()
+jdbc_hook.set_autocommit(jdbc_conn, False)
+
@patch("airflow.providers.jdbc.hooks.jdbc.jaydebeapi.connect")
def test_jdbc_conn_get_autocommit(self, _):
jdbc_hook = JdbcHook()
@@ -89,6 +99,13 @@ class TestJdbcHook:
jdbc_hook.get_autocommit(jdbc_conn)
jdbc_conn.jconn.getAutoCommit.assert_called_once_with()
+@patch("airflow.providers.jdbc.hooks.jdbc.jaydebeapi.connect")
+def
test_jdbc_conn_get_autocommit_when_not_supported_then_return_false(self, _):
+jdbc_hook = JdbcHook()
+jdbc_conn = jdbc_hook.get_conn()
+jdbc_conn.jconn.getAutoCommit.side_effect = jaydebeapi.Error()
+assert jdbc_hook.get_autocommit(jdbc_conn) is False
+
def test_driver_hook_params(self):
hook = get_hook(hook_params=dict(driver_path="Blah driver path",
driver_class="Blah driver class"))
assert hook.driver_path == "Blah driver path"
@@ -161,3 +178,12 @@ class TestJdbcHook:
"have supplied 'driver_class' via connection extra but it
will not be u
(airflow) branch main updated: Always use the executemany method when inserting rows in DbApiHook as it's way much faster (#38715)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 7ab24c7723 Always use the executemany method when inserting rows in
DbApiHook as it's way much faster (#38715)
7ab24c7723 is described below
commit 7ab24c7723c65c90626b10db63444b88c0380e14
Author: David Blain
AuthorDate: Fri Apr 12 11:43:23 2024 +0200
Always use the executemany method when inserting rows in DbApiHook as it's
way much faster (#38715)
-
Co-authored-by: David Blain
Co-authored-by: Tzu-ping Chung
---
airflow/providers/common/sql/hooks/sql.py | 52 ++---
airflow/providers/common/sql/hooks/sql.pyi | 1 +
airflow/providers/odbc/hooks/odbc.py| 1 +
airflow/providers/postgres/hooks/postgres.py| 1 +
airflow/providers/teradata/hooks/teradata.py| 48 ---
tests/deprecations_ignore.yml | 6 +++
tests/providers/common/sql/hooks/test_dbapi.py | 3 +-
tests/providers/postgres/hooks/test_postgres.py | 9 ++---
tests/providers/teradata/hooks/test_teradata.py | 22 ++-
9 files changed, 73 insertions(+), 70 deletions(-)
diff --git a/airflow/providers/common/sql/hooks/sql.py
b/airflow/providers/common/sql/hooks/sql.py
index 7f1536a39b..4625c2e014 100644
--- a/airflow/providers/common/sql/hooks/sql.py
+++ b/airflow/providers/common/sql/hooks/sql.py
@@ -18,7 +18,7 @@ from __future__ import annotations
import contextlib
import warnings
-from contextlib import closing
+from contextlib import closing, contextmanager
from datetime import datetime
from typing import (
TYPE_CHECKING,
@@ -147,6 +147,8 @@ class DbApiHook(BaseHook):
default_conn_name = "default_conn_id"
# Override if this db supports autocommit.
supports_autocommit = False
+# Override if this db supports executemany.
+supports_executemany = False
# Override with the object that exposes the connect method
connector: ConnectorProtocol | None = None
# Override with db-specific query to check connection
@@ -408,10 +410,7 @@ class DbApiHook(BaseHook):
else:
raise ValueError("List of SQL statements is empty")
_last_result = None
-with closing(self.get_conn()) as conn:
-if self.supports_autocommit:
-self.set_autocommit(conn, autocommit)
-
+with self._create_autocommit_connection(autocommit) as conn:
with closing(conn.cursor()) as cur:
results = []
for sql_statement in sql_list:
@@ -528,6 +527,14 @@ class DbApiHook(BaseHook):
return self._replace_statement_format.format(table, target_fields,
",".join(placeholders))
+@contextmanager
+def _create_autocommit_connection(self, autocommit: bool = False):
+"""Context manager that closes the connection after use and detects if
autocommit is supported."""
+with closing(self.get_conn()) as conn:
+if self.supports_autocommit:
+self.set_autocommit(conn, autocommit)
+yield conn
+
def insert_rows(
self,
table,
@@ -550,47 +557,48 @@ class DbApiHook(BaseHook):
:param commit_every: The maximum number of rows to insert in one
transaction. Set to 0 to insert all rows in one transaction.
:param replace: Whether to replace instead of insert
-:param executemany: Insert all rows at once in chunks defined by the
commit_every parameter, only
-works if all rows have same number of column names but leads to
better performance
+:param executemany: (Deprecated) If True, all rows are inserted at
once in
+chunks defined by the commit_every parameter. This only works if
all rows
+have same number of column names, but leads to better performance.
"""
-i = 0
-with closing(self.get_conn()) as conn:
-if self.supports_autocommit:
-self.set_autocommit(conn, False)
+if executemany:
+warnings.warn(
+"executemany parameter is deprecated, override
supports_executemany instead.",
+AirflowProviderDeprecationWarning,
+stacklevel=2,
+)
+with self._create_autocommit_connection() as conn:
conn.commit()
-
with closing(conn.cursor()) as cur:
-if executemany:
+if self.supports_executemany or executemany:
for chunked_rows in chunked(rows, commit_every):
values = list(
map(
-lambda row: tuple(map(lambda cell:
self.
(airflow) branch main updated: Limit redis image to 7.2 in sample docker-compose.yaml (#38938)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new cd88ddb4f1 Limit redis image to 7.2 in sample docker-compose.yaml (#38938) cd88ddb4f1 is described below commit cd88ddb4f1617e3d9855e8927d3d7aa4b6daea9a Author: Andrey Anshin AuthorDate: Thu Apr 11 22:58:47 2024 +0400 Limit redis image to 7.2 in sample docker-compose.yaml (#38938) --- docs/apache-airflow/howto/docker-compose/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml index 9effc2d504..92b0eab937 100644 --- a/docs/apache-airflow/howto/docker-compose/docker-compose.yaml +++ b/docs/apache-airflow/howto/docker-compose/docker-compose.yaml @@ -102,7 +102,9 @@ services: restart: always redis: -image: redis:latest +# Redis is limited to 7.2-bookworm due to licencing change +# https://redis.io/blog/redis-adopts-dual-source-available-licensing/ +image: redis:7.2-bookworm expose: - 6379 healthcheck:
(airflow) branch main updated: Replace also "-" in providers compatibility check specification (#38936)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 55b251f849 Replace also "-" in providers compatibility check
specification (#38936)
55b251f849 is described below
commit 55b251f84936f3f893fc01224ea8d718b72cfd0f
Author: Jarek Potiuk
AuthorDate: Thu Apr 11 19:02:06 2024 +0200
Replace also "-" in providers compatibility check specification (#38936)
Just be a little more resilient and also accept "-" in the short name
---
dev/breeze/src/airflow_breeze/global_constants.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index e17c60ad0d..b527cafe3c 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -464,7 +464,9 @@ CHICKEN_EGG_PROVIDERS = " ".join([])
def _exclusion(providers: Iterable[str]) -> str:
-return " ".join([f"apache_airflow_providers_{provider.replace('.', '_')}*"
for provider in providers])
+return " ".join(
+[f"apache_airflow_providers_{provider.replace('.',
'_').replace('-','_')}*" for provider in providers]
+)
BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str]] = [
(airflow) branch main updated: Update build dependencies to their latest versions (#38930)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/main by this push: new 3af00fa2a2 Update build dependencies to their latest versions (#38930) 3af00fa2a2 is described below commit 3af00fa2a206e31a449648e33eed1f806d8b62b7 Author: Jarek Potiuk AuthorDate: Thu Apr 11 15:37:19 2024 +0200 Update build dependencies to their latest versions (#38930) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f60a702142..985acc75fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ requires = [ "pluggy==1.4.0", "smmap==5.0.1", "tomli==2.0.1; python_version < '3.11'", -"trove-classifiers==2024.3.25", +"trove-classifiers==2024.4.10", ] build-backend = "hatchling.build"
(airflow) branch main updated (e4424aebcc -> 486522cadc)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from e4424aebcc Raise an error in case of `pytest.PytestReturnNotNoneWarning` (#38923) add 486522cadc Limit redis image to 7.2 in our chart (#38928) No new revisions were added by this update. Summary of changes: chart/values.schema.json | 2 +- chart/values.yaml| 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-)
(airflow) branch main updated: Enhancement for SSL-support in CloudSQLExecuteQueryOperator (#38894)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1757704d8f Enhancement for SSL-support in CloudSQLExecuteQueryOperator
(#38894)
1757704d8f is described below
commit 1757704d8f7c7a335cdf8f90ccb12b4d8e6f9d9a
Author: max <42827971+moiseen...@users.noreply.github.com>
AuthorDate: Thu Apr 11 10:56:34 2024 +0200
Enhancement for SSL-support in CloudSQLExecuteQueryOperator (#38894)
---
.../cloud/example_dags/example_cloud_sql_query.py | 289 ---
airflow/providers/google/cloud/hooks/cloud_sql.py | 152 +-
.../providers/google/cloud/hooks/secret_manager.py | 252 -
.../providers/google/cloud/operators/cloud_sql.py | 57 +-
.../operators/cloud/cloud_sql.rst | 54 +-
docs/spelling_wordlist.txt | 8 +
.../providers/google/cloud/hooks/test_cloud_sql.py | 417 ++-
.../google/cloud/hooks/test_secret_manager.py | 239 -
.../cloud/cloud_sql/example_cloud_sql_query.py | 572 +
.../cloud_sql/example_cloud_sql_query_mysql.py | 285 --
.../cloud_sql/example_cloud_sql_query_postgres.py | 290 ---
.../cloud/cloud_sql/example_cloud_sql_query_ssl.py | 518 +++
12 files changed, 2223 insertions(+), 910 deletions(-)
diff --git
a/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
b/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
deleted file mode 100644
index b883ed13cf..00
--- a/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
+++ /dev/null
@@ -1,289 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example Airflow DAG that performs query in a Cloud SQL instance.
-
-This DAG relies on the following OS environment variables
-
-* GCP_PROJECT_ID - Google Cloud project for the Cloud SQL instance
-* GCP_REGION - Google Cloud region where the database is created
-*
-* GCSQL_POSTGRES_INSTANCE_NAME - Name of the postgres Cloud SQL instance
-* GCSQL_POSTGRES_USER - Name of the postgres database user
-* GCSQL_POSTGRES_PASSWORD - Password of the postgres database user
-* GCSQL_POSTGRES_PUBLIC_IP - Public IP of the Postgres database
-* GCSQL_POSTGRES_PUBLIC_PORT - Port of the postgres database
-*
-* GCSQL_MYSQL_INSTANCE_NAME - Name of the postgres Cloud SQL instance
-* GCSQL_MYSQL_USER - Name of the mysql database user
-* GCSQL_MYSQL_PASSWORD - Password of the mysql database user
-* GCSQL_MYSQL_PUBLIC_IP - Public IP of the mysql database
-* GCSQL_MYSQL_PUBLIC_PORT - Port of the mysql database
-"""
-
-from __future__ import annotations
-
-import os
-import subprocess
-from datetime import datetime
-from pathlib import Path
-from urllib.parse import quote_plus
-
-from airflow.models.dag import DAG
-from airflow.providers.google.cloud.operators.cloud_sql import
CloudSQLExecuteQueryOperator
-
-GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-GCP_REGION = os.environ.get("GCP_REGION", "europe-west1")
-
-GCSQL_POSTGRES_INSTANCE_NAME_QUERY = os.environ.get(
-"GCSQL_POSTGRES_INSTANCE_NAME_QUERY", "test-postgres-query"
-)
-GCSQL_POSTGRES_DATABASE_NAME = os.environ.get("GCSQL_POSTGRES_DATABASE_NAME",
"postgresdb")
-GCSQL_POSTGRES_USER = os.environ.get("GCSQL_POSTGRES_USER", "postgres_user")
-GCSQL_POSTGRES_PASSWORD = os.environ.get("GCSQL_POSTGRES_PASSWORD",
"JoxHlwrPzwch0gz9")
-GCSQL_POSTGRES_PUBLIC_IP = os.environ.get("GCSQL_POSTGRES_PUBLIC_IP",
"0.0.0.0")
-GCSQL_POSTGRES_PUBLIC_PORT = os.environ.get("GCSQL_POSTGRES_PUBLIC_PORT", 5432)
-GCSQL_POSTGRES_CLIENT_CERT_FILE = os.environ.get(
-"GCSQL_POSTGRES_CLIENT_CERT_FILE", ".key/postgres-client-cert.pem"
-)
-GCSQL_POSTGRES_CLIENT_KEY_FILE = os.environ.get(
-"GCSQL_POSTGRES_CLIENT_KEY_FILE", ".key/postgres-client-key.pem"
-)
-GCSQL_POSTGRES_SE
(airflow) branch main updated (b90fa78e88 -> 4f169bd2f3)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from b90fa78e88 Fix check of correct dag when remote call for _get_ti (#38909) add 4f169bd2f3 Fix `update-common-sql-api-stubs` pre-commit check (#38915) No new revisions were added by this update. Summary of changes: airflow/providers/common/sql/hooks/sql.pyi | 85 -- airflow/providers/common/sql/operators/sql.pyi | 100 - airflow/providers/common/sql/sensors/sql.pyi | 18 ++-- .../ci/pre_commit/update_common_sql_api_stubs.py | 21 +++-- 4 files changed, 137 insertions(+), 87 deletions(-)
(airflow) branch main updated (34878507ef -> 1608231993)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 34878507ef Widen cheat sheet column to avoid wrapping commands (#3)
add 1608231993 Add multi-team diagrams (#38861)
No new revisions were added by this update.
Summary of changes:
..._team_airflow_architecture_with_grpc_api.md5sum | 1 +
...lti_team_airflow_architecture_with_grpc_api.png | Bin 0 -> 647436 bytes
...lti_team_airflow_architecture_with_grpc_api.py} | 38 -
...am_airflow_architecture_without_grpc_api.md5sum | 1 +
..._team_airflow_architecture_without_grpc_api.png | Bin 0 -> 597552 bytes
..._team_airflow_architecture_without_grpc_api.py} | 92 +++--
...diagram_multitenant_airflow_architecture.md5sum | 1 -
.../diagram_multitenant_airflow_architecture.png | Bin 601270 -> 0 bytes
hatch_build.py | 1 +
9 files changed, 52 insertions(+), 82 deletions(-)
create mode 100644
docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.md5sum
create mode 100644
docs/apache-airflow/img/diagram_multi_team_airflow_architecture_with_grpc_api.png
copy docs/apache-airflow/img/{diagram_multitenant_airflow_architecture.py =>
diagram_multi_team_airflow_architecture_with_grpc_api.py} (87%)
create mode 100644
docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.md5sum
create mode 100644
docs/apache-airflow/img/diagram_multi_team_airflow_architecture_without_grpc_api.png
rename docs/apache-airflow/img/{diagram_multitenant_airflow_architecture.py =>
diagram_multi_team_airflow_architecture_without_grpc_api.py} (73%)
delete mode 100644
docs/apache-airflow/img/diagram_multitenant_airflow_architecture.md5sum
delete mode 100644
docs/apache-airflow/img/diagram_multitenant_airflow_architecture.png
(airflow-site) branch main updated: [Landing pages] Replace use case video iframes with links (#1002)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow-site.git
The following commit(s) were added to refs/heads/main by this push:
new b6ce741de7 [Landing pages] Replace use case video iframes with links
(#1002)
b6ce741de7 is described below
commit b6ce741de7743dc0b60d16dc875d5f9cf6ff891a
Author: Michael Robinson <68482867+merobi-...@users.noreply.github.com>
AuthorDate: Wed Apr 10 10:04:13 2024 -0400
[Landing pages] Replace use case video iframes with links (#1002)
* Fix for google news error.
Signed-off-by: merobi-hub
* Replace use case video ifrmaes with links.
Signed-off-by: merobi-hub
-
Signed-off-by: merobi-hub
---
.../content/en/use-cases/business_operations.md| 22 +++---
.../site/content/en/use-cases/etl_analytics.md | 21 +++--
.../en/use-cases/infrastructure-management.md | 21 +++--
landing-pages/site/content/en/use-cases/mlops.md | 21 +++--
.../site/layouts/partials/hooks/head-end.html | 3 ---
5 files changed, 12 insertions(+), 76 deletions(-)
diff --git a/landing-pages/site/content/en/use-cases/business_operations.md
b/landing-pages/site/content/en/use-cases/business_operations.md
index 5daf45cc19..8c829ae8f9 100644
--- a/landing-pages/site/content/en/use-cases/business_operations.md
+++ b/landing-pages/site/content/en/use-cases/business_operations.md
@@ -20,27 +20,11 @@ This video shows an example of using Airflow to run the
pipelines that power a c
-
+https://www.youtube.com/embed/2CEApKN0z1U?autoplay=1;>
+
+
-
-document.getElementById('videoPlaceholder').addEventListener('click',
function() {
-var iframe = document.createElement('iframe');
-iframe.setAttribute('src',
'<a rel="nofollow" href="https://www.youtube.com/embed/2CEApKN0z1U?autoplay=1">https://www.youtube.com/embed/2CEApKN0z1U?autoplay=1</a>');
-iframe.setAttribute('width', '100%');
-iframe.setAttribute('height', '315');
-iframe.setAttribute('frameborder', '0');
-iframe.setAttribute('allow', 'accelerometer; autoplay;
clipboard-write; encrypted-media; gyroscope; picture-in-picture');
-iframe.setAttribute('allowfullscreen', 'allowfullscreen');
-iframe.style.display = 'block';
-
-var videoContainer = document.getElementById('videoContainer');
-videoContainer.innerHTML = '';
-videoContainer.appendChild(iframe);
-});
-
-
-
## Why use Airflow for Business Operations?
diff --git a/landing-pages/site/content/en/use-cases/etl_analytics.md
b/landing-pages/site/content/en/use-cases/etl_analytics.md
index b0f612cfbb..c578c3e22c 100644
--- a/landing-pages/site/content/en/use-cases/etl_analytics.md
+++ b/landing-pages/site/content/en/use-cases/etl_analytics.md
@@ -19,26 +19,11 @@ The video below shows a simple ETL/ELT pipeline in Airflow
that extracts climate
-
+https://www.youtube.com/embed/ljBU_VyihVQ?autoplay=1;>
+
+
-
-document.getElementById('videoPlaceholder').addEventListener('click',
function() {
-var iframe = document.createElement('iframe');
-iframe.setAttribute('src',
'<a rel="nofollow" href="https://www.youtube.com/embed/ljBU_VyihVQ?autoplay=1">https://www.youtube.com/embed/ljBU_VyihVQ?autoplay=1</a>');
-iframe.setAttribute('width', '90%');
-iframe.setAttribute('height', '315');
-iframe.setAttribute('frameborder', '0');
-iframe.setAttribute('allow', 'accelerometer; autoplay;
clipboard-write; encrypted-media; gyroscope; picture-in-picture');
-iframe.setAttribute('allowfullscreen', 'allowfullscreen');
-iframe.style.display = 'block';
-
-var videoContainer = document.getElementById('videoContainer');
-videoContainer.innerHTML = '';
-videoContainer.appendChild(iframe);
-});
-
-
## Why use Airflow for ETL/ELT pipelines?
diff --git
a/landing-pages/site/content/en/use-cases/infrastructure-management.md
b/landing-pages/site/content/en/use-cases/infrastructure-management.md
index bab2aea859..60adf5543e 100644
--- a/landing-pages/site/content/en/use-cases/infrastructure-management.md
+++ b/landing-pages/site/content/en/use-cases/infrastructure-management.md
@@ -18,26 +18,11 @@ Airflow can interact with any API, which makes it a great
tool to manage your in
Infrastructure management is often needed within the context of other use
cases, such as MLOps, or implementing data quality checks. This video shows an
example of how it might be used for an MLOps pipeline. You can find the code
shown in this example
[here](https://github.com/astronomer/use-case-setup-teardown-data-q
(airflow) branch main updated: Add Airflow 2.8.0 to providers compatibility checks (#38878)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2b7c47cf47 Add Airflow 2.8.0 to providers compatibility checks (#38878)
2b7c47cf47 is described below
commit 2b7c47cf47dd89e6dd25d3ac643cbd4001aa4a86
Author: Andrey Anshin
AuthorDate: Wed Apr 10 17:05:16 2024 +0400
Add Airflow 2.8.0 to providers compatibility checks (#38878)
---
dev/breeze/src/airflow_breeze/global_constants.py | 5 +
1 file changed, 5 insertions(+)
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 32c8eb98f0..e17c60ad0d 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -478,6 +478,11 @@ BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str]]
= [
"airflow-version": "2.7.1",
"remove-providers": _exclusion(["common.io", "fab"]),
},
+{
+"python-version": "3.8",
+"airflow-version": "2.8.0",
+"remove-providers": _exclusion(["fab"]),
+},
]
(airflow) branch main updated: Use celery worker CLI from Airflow package for Airflow < 2.8.0 (#38879)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 3e30b3a025 Use celery worker CLI from Airflow package for Airflow <
2.8.0 (#38879)
3e30b3a025 is described below
commit 3e30b3a02584e13fa130255b25756eaf7dfe35d3
Author: Jarek Potiuk
AuthorDate: Tue Apr 9 21:44:59 2024 +0200
Use celery worker CLI from Airflow package for Airflow < 2.8.0 (#38879)
Celery provider has an ambedded Airflow CLI command as of 3.6.1. When
the #36794 was merged, we thought mistakenly that it will only be used
in airflow 2.9.0+, so we used a feature introduced in Airflow 2.8.0 in
the #34945 - but in fact the CLI command is configured by the Celery
Executor which is also part of the Celery provider, so it was also
used for airflow < 2.8.0 and failed due to missing import.
This PR checks if Airflow version is < 2.8.0 and if so, it falls back
to built-in airflow CLI command.
---
airflow/providers/celery/executors/celery_executor.py | 16 +---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/airflow/providers/celery/executors/celery_executor.py
b/airflow/providers/celery/executors/celery_executor.py
index 2a75be91da..0b4293cde7 100644
--- a/airflow/providers/celery/executors/celery_executor.py
+++ b/airflow/providers/celery/executors/celery_executor.py
@@ -30,10 +30,12 @@ import operator
import time
from collections import Counter
from concurrent.futures import ProcessPoolExecutor
+from importlib.metadata import version as importlib_version
from multiprocessing import cpu_count
from typing import TYPE_CHECKING, Any, Optional, Sequence, Tuple
from celery import states as celery_states
+from packaging.version import Version
try:
from airflow.cli.cli_config import (
@@ -178,11 +180,19 @@ ARG_WITHOUT_GOSSIP = Arg(
action="store_true",
)
+AIRFLOW_VERSION = Version(importlib_version("apache-airflow"))
+
+CELERY_CLI_COMMAND_PATH = (
+"airflow.providers.celery.cli.celery_command"
+if AIRFLOW_VERSION >= Version("2.8.0")
+else "airflow.cli.commands.celery_command"
+)
+
CELERY_COMMANDS = (
ActionCommand(
name="worker",
help="Start a Celery worker node",
-
func=lazy_load_command("airflow.providers.celery.cli.celery_command.worker"),
+func=lazy_load_command(f"{CELERY_CLI_COMMAND_PATH}.worker"),
args=(
ARG_QUEUES,
ARG_CONCURRENCY,
@@ -203,7 +213,7 @@ CELERY_COMMANDS = (
ActionCommand(
name="flower",
help="Start a Celery Flower",
-
func=lazy_load_command("airflow.providers.celery.cli.celery_command.flower"),
+func=lazy_load_command(f"{CELERY_CLI_COMMAND_PATH}.flower"),
args=(
ARG_FLOWER_HOSTNAME,
ARG_FLOWER_PORT,
@@ -222,7 +232,7 @@ CELERY_COMMANDS = (
ActionCommand(
name="stop",
help="Stop the Celery worker gracefully",
-
func=lazy_load_command("airflow.providers.celery.cli.celery_command.stop_worker"),
+func=lazy_load_command(f"{CELERY_CLI_COMMAND_PATH}.stop_worker"),
args=(ARG_PID, ARG_VERBOSE),
),
)
(airflow) branch main updated: Improve editable airflow installation by adding preinstalled deps (#38764)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d409b8be96 Improve editable airflow installation by adding
preinstalled deps (#38764)
d409b8be96 is described below
commit d409b8be966b51207e10129e7709e943a5dc7ac3
Author: Jarek Potiuk
AuthorDate: Mon Apr 8 18:56:17 2024 +0200
Improve editable airflow installation by adding preinstalled deps (#38764)
* Improve editable airflow installation by adding preinstalled deps
While preparing for presentation about modernizing packaging setup
I found out that we can slightly improve editable installation of
airflow. So far we required a "devel" installation in order to make
a working editable installation of airflow (because the devel extra
was installing dependencies for all the pre-installed providers).
However that required to synchronize the list of providers installed
in `devel` dependency with the list of preinstalled providers, as
well as it made `pip install -e .` resulting with a non-working
version of Airflow (because it requires `fab` provider dependencies
to start airflow webserver).
This PR improves it - in editable mode, instead of adding the
pre-installed providers, we add their dependencies. This way we can
remove the pre-installed providers from the list of devel providers
to install - because now the pre-installed provider dependencies
are installed simply as "required" dependencies.
As a result - simple `pip install -e .` should now result in fully
working airflow installation - without all the devel goodies and
without celery and kubernetes dependencies, but fully usable for
sequential and local executor cases.
Also reviewed and updated the comments in hatch_build.py to better
reflect the purpose and behaviour of some of the methods there.
* Update hatch_build.py
Co-authored-by: Ephraim Anierobi
-
Co-authored-by: Ephraim Anierobi
---
airflow_pre_installed_providers.txt | 9 ---
hatch_build.py | 150 ++--
2 files changed, 110 insertions(+), 49 deletions(-)
diff --git a/airflow_pre_installed_providers.txt
b/airflow_pre_installed_providers.txt
deleted file mode 100644
index e717ea696e..00
--- a/airflow_pre_installed_providers.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# List of all the providers that are pre-installed when you run `pip install
apache-airflow` without extras
-common.io
-common.sql
-fab>=1.0.2rc1
-ftp
-http
-imap
-smtp
-sqlite
diff --git a/hatch_build.py b/hatch_build.py
index 69b6da7a1c..532ee7d3e3 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -273,8 +273,6 @@ DEVEL_EXTRAS: dict[str, list[str]] = {
"devel": [
"apache-airflow[celery]",
"apache-airflow[cncf-kubernetes]",
-"apache-airflow[common-io]",
-"apache-airflow[common-sql]",
"apache-airflow[devel-debuggers]",
"apache-airflow[devel-devscripts]",
"apache-airflow[devel-duckdb]",
@@ -282,11 +280,6 @@ DEVEL_EXTRAS: dict[str, list[str]] = {
"apache-airflow[devel-sentry]",
"apache-airflow[devel-static-checks]",
"apache-airflow[devel-tests]",
-"apache-airflow[fab]",
-"apache-airflow[ftp]",
-"apache-airflow[http]",
-"apache-airflow[imap]",
-"apache-airflow[sqlite]",
],
"devel-all-dbs": [
"apache-airflow[apache-cassandra]",
@@ -550,11 +543,35 @@ ALL_DYNAMIC_EXTRAS: list[str] = sorted(
def get_provider_id(provider_spec: str) -> str:
-# in case provider_spec is "="
-return provider_spec.split(">=")[0]
+"""
+Extract provider id from provider specification.
+
+:param provider_spec: provider specification can be in the form of the
"PROVIDER_ID" or
+ "apache-airflow-providers-PROVIDER", optionally followed by
">=VERSION".
+
+:return: short provider_id with `.` instead of `-` in case of `apache` and
other providers with
+ `-` in the name.
+"""
+_provider_id = provider_spec.split(">=")[0]
+if _provider_id.startswith("apache-airflow-providers-"):
+_provider_id = _provider_id.replace("apache-airflow-providers-",
"").replace("-", ".")
+return _provider_id
def get_provider_requirement(provider_spec: str) -> str:
+"""
+Convert provider specification with provider_i
(airflow) branch main updated: Remove decorator from rendering fields example (#38827)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 285f037dbc Remove decorator from rendering fields example (#38827)
285f037dbc is described below
commit 285f037dbcc0f5e23c2d6ac99bfd6cab86c96ac3
Author: Elad Kalif <45845474+elad...@users.noreply.github.com>
AuthorDate: Mon Apr 8 19:09:17 2024 +0300
Remove decorator from rendering fields example (#38827)
* Remove decorator from rendering fields example
* fix
---
docs/apache-airflow/core-concepts/operators.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/apache-airflow/core-concepts/operators.rst
b/docs/apache-airflow/core-concepts/operators.rst
index c7193789bc..0330ef43fb 100644
--- a/docs/apache-airflow/core-concepts/operators.rst
+++ b/docs/apache-airflow/core-concepts/operators.rst
@@ -246,9 +246,9 @@ you can pass ``render_template_as_native_obj=True`` to the
DAG as follows:
return json.loads(data_string)
-@task(task_id="transform")
def transform(order_data):
print(type(order_data))
+total_order_value = 0
for value in order_data.values():
total_order_value += value
return {"total_order_value": total_order_value}
(airflow) branch main updated (c81449372e -> f50cc6180c)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from c81449372e Ignore subdag operator warnings during the (re)initialise DB in tests (#38823) add f50cc6180c Workaround occasional failures of docker system prune in public runners (#38825) No new revisions were added by this update. Summary of changes: scripts/ci/cleanup_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(airflow) branch main updated: Fix 'use case' spelling (#38817)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 4cd15f8926 Fix 'use case' spelling (#38817)
4cd15f8926 is described below
commit 4cd15f8926b6aa2b25dbef3c961d545d6cbf72be
Author: Tzu-ping Chung
AuthorDate: Mon Apr 8 15:27:04 2024 +0800
Fix 'use case' spelling (#38817)
The top-level site uses 'use case' but the documentation uses
'use-case'. The former seems to be the more popular spelling.
---
docs/conf.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/conf.py b/docs/conf.py
index 91b4f23996..00e698803c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -382,7 +382,7 @@ html_theme_options["navbar_links"] = [
{"href": "/community/", "text": "Community"},
{"href": "/meetups/", "text": "Meetups"},
{"href": "/docs/", "text": "Documentation"},
-{"href": "/use-cases/", "text": "Use-cases"},
+{"href": "/use-cases/", "text": "Use Cases"},
{"href": "/announcements/", "text": "Announcements"},
{"href": "/blog/", "text": "Blog"},
{"href": "/ecosystem/", "text": "Ecosystem"},
(airflow) branch main updated (472b646acb -> ef97ed245b)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 472b646acb fix: skip apache beam pipeline options if value is set to false (#38496) add ef97ed245b Add hook_params to template_fields for BaseSQLOperator-related Operators (#38724) No new revisions were added by this update. Summary of changes: airflow/providers/common/sql/operators/sql.py| 13 +++-- airflow/providers/common/sql/sensors/sql.py | 2 +- tests/providers/common/sql/operators/test_sql.py | 13 + 3 files changed, 21 insertions(+), 7 deletions(-)
(airflow) branch main updated (6d3d2075ae -> 472b646acb)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 6d3d2075ae fix: try002 for provider fab (#38801) add 472b646acb fix: skip apache beam pipeline options if value is set to false (#38496) No new revisions were added by this update. Summary of changes: airflow/providers/apache/beam/hooks/beam.py| 2 ++ tests/providers/apache/beam/hooks/test_beam.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-)
(airflow) branch main updated (153370310e -> 6d3d2075ae)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 153370310e fix: try002 for provider airbyte (#38786) add 6d3d2075ae fix: try002 for provider fab (#38801) No new revisions were added by this update. Summary of changes: airflow/providers/fab/auth_manager/fab_auth_manager.py | 4 ++-- airflow/providers/fab/auth_manager/security_manager/override.py | 2 +- pyproject.toml | 3 --- tests/providers/fab/auth_manager/test_fab_auth_manager.py | 5 +++-- 4 files changed, 6 insertions(+), 8 deletions(-)
(airflow) branch main updated: fix: try002 for provider airbyte (#38786)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 153370310e fix: try002 for provider airbyte (#38786)
153370310e is described below
commit 153370310ed2c4b0687cd350ca8014c802f20045
Author: Sebastian Daum
AuthorDate: Sun Apr 7 21:55:58 2024 +0200
fix: try002 for provider airbyte (#38786)
---
airflow/providers/airbyte/hooks/airbyte.py | 2 +-
airflow/providers/airbyte/operators/airbyte.py | 4 +++-
airflow/providers/airbyte/sensors/airbyte.py | 4 +++-
pyproject.toml | 4
tests/providers/airbyte/hooks/test_airbyte.py | 2 +-
5 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/airflow/providers/airbyte/hooks/airbyte.py
b/airflow/providers/airbyte/hooks/airbyte.py
index e0ae41ff13..7c1132fa57 100644
--- a/airflow/providers/airbyte/hooks/airbyte.py
+++ b/airflow/providers/airbyte/hooks/airbyte.py
@@ -168,7 +168,7 @@ class AirbyteHook(HttpHook):
elif state == self.CANCELLED:
raise AirflowException(f"Job was cancelled:\n{job}")
else:
-raise Exception(f"Encountered unexpected state `{state}` for
job_id `{job_id}`")
+raise AirflowException(f"Encountered unexpected state
`{state}` for job_id `{job_id}`")
def submit_sync_connection(self, connection_id: str) -> Any:
"""
diff --git a/airflow/providers/airbyte/operators/airbyte.py
b/airflow/providers/airbyte/operators/airbyte.py
index c2f0a56202..0dd82d4683 100644
--- a/airflow/providers/airbyte/operators/airbyte.py
+++ b/airflow/providers/airbyte/operators/airbyte.py
@@ -116,7 +116,9 @@ class AirbyteTriggerSyncOperator(BaseOperator):
elif state == hook.CANCELLED:
raise AirflowException(f"Job was
cancelled:\n{self.job_id}")
else:
-raise Exception(f"Encountered unexpected state `{state}`
for job_id `{self.job_id}")
+raise AirflowException(
+f"Encountered unexpected state `{state}` for job_id
`{self.job_id}"
+)
else:
hook.wait_for_job(job_id=self.job_id,
wait_seconds=self.wait_seconds, timeout=self.timeout)
self.log.info("Job %s completed successfully", self.job_id)
diff --git a/airflow/providers/airbyte/sensors/airbyte.py
b/airflow/providers/airbyte/sensors/airbyte.py
index 0ccdd903c4..35e552b2d6 100644
--- a/airflow/providers/airbyte/sensors/airbyte.py
+++ b/airflow/providers/airbyte/sensors/airbyte.py
@@ -150,7 +150,9 @@ class AirbyteJobSensor(BaseSensorOperator):
elif state == hook.CANCELLED:
raise AirflowException(f"Job was cancelled:\n{job}")
else:
-raise Exception(f"Encountered unexpected state `{state}` for
job_id `{self.airbyte_job_id}")
+raise AirflowException(
+f"Encountered unexpected state `{state}` for job_id
`{self.airbyte_job_id}"
+)
def execute_complete(self, context: Context, event: Any = None) -> None:
"""
diff --git a/pyproject.toml b/pyproject.toml
index 5790943f53..1bc10f7ec7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -379,10 +379,6 @@ combine-as-imports = true
"tests/providers/snowflake/operators/test_snowflake_sql.py" = ["E402"]
# All the providers modules which do not follow TRY002 yet
-# airbyte
-"airflow/providers/airbyte/hooks/airbyte.py" = ["TRY002"]
-"airflow/providers/airbyte/operators/airbyte.py" = ["TRY002"]
-"airflow/providers/airbyte/sensors/airbyte.py" = ["TRY002"]
# amazon
"airflow/providers/amazon/aws/operators/athena.py" = ["TRY002"]
"airflow/providers/amazon/aws/operators/emr.py" = ["TRY002"]
diff --git a/tests/providers/airbyte/hooks/test_airbyte.py
b/tests/providers/airbyte/hooks/test_airbyte.py
index 400741820b..18d935bda3 100644
--- a/tests/providers/airbyte/hooks/test_airbyte.py
+++ b/tests/providers/airbyte/hooks/test_airbyte.py
@@ -133,7 +133,7 @@ class TestAirbyteHook:
self.return_value_get_job(self.hook.RUNNING),
self.return_value_get_job("UNRECOGNIZED"),
]
-with pytest.raises(Exception, match="unexpected state"):
+with pytest.raises(AirflowException, match="unexpected state"):
self.hook.wait_for_job(job_id=self.job_id, wait_seconds=0)
calls = [mock.call(job_id=self.job_id), mock.call(job_id=self.job_id)]
(airflow) branch main updated (395a9ab24d -> 9fcc1e826e)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 395a9ab24d Add a newsfragment for PR 38015 validation (#38811)
add 9fcc1e826e Add pre-commit script to keep tags sorted in OpenAPI spec
(#38813)
No new revisions were added by this update.
Summary of changes:
.pre-commit-config.yaml| 8 +
contributing-docs/08_static_code_checks.rst| 2 ++
dev/breeze/doc/images/output_static-checks.svg | 8 ++---
dev/breeze/doc/images/output_static-checks.txt | 2 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py| 1 +
...d_providers.py => sort_tags_in_openapi_spec.py} | 40 --
6 files changed, 45 insertions(+), 16 deletions(-)
copy scripts/ci/pre_commit/{sort_installed_providers.py =>
sort_tags_in_openapi_spec.py} (51%)
(airflow) branch main updated (5ab7033272 -> 6004175d55)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git from 5ab7033272 Add MobiKwik as an Airflow user into INTHEWILD.md (#38812) add 6004175d55 (img/edge_label_example.png): changed imaged to new label view (#38802) No new revisions were added by this update. Summary of changes: docs/apache-airflow/img/edge_label_example.png | Bin 33270 -> 37438 bytes 1 file changed, 0 insertions(+), 0 deletions(-)
(airflow) branch constraints-2-9 updated: Upgrade constraints to rc1
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch constraints-2-9 in repository https://gitbox.apache.org/repos/asf/airflow.git The following commit(s) were added to refs/heads/constraints-2-9 by this push: new 4642ff83ce Upgrade constraints to rc1 4642ff83ce is described below commit 4642ff83ce708c9f279bd8170787855c5be73f31 Author: Jarek Potiuk AuthorDate: Sat Apr 6 23:33:11 2024 +0200 Upgrade constraints to rc1 --- constraints-3.10.txt | 2 +- constraints-3.11.txt | 2 +- constraints-3.12.txt | 2 +- constraints-3.8.txt | 2 +- constraints-3.9.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/constraints-3.10.txt b/constraints-3.10.txt index de912943c5..4dff997ac2 100644 --- a/constraints-3.10.txt +++ b/constraints-3.10.txt @@ -128,7 +128,7 @@ apache-airflow-providers-discord==3.6.0 apache-airflow-providers-docker==3.9.2 apache-airflow-providers-elasticsearch==5.3.3 apache-airflow-providers-exasol==4.4.3 -apache-airflow-providers-fab==1.0.2 +apache-airflow-providers-fab==1.0.2rc1 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1 diff --git a/constraints-3.11.txt b/constraints-3.11.txt index ae32549cf0..355a60dedc 100644 --- a/constraints-3.11.txt +++ b/constraints-3.11.txt @@ -128,7 +128,7 @@ apache-airflow-providers-discord==3.6.0 apache-airflow-providers-docker==3.9.2 apache-airflow-providers-elasticsearch==5.3.3 apache-airflow-providers-exasol==4.4.3 -apache-airflow-providers-fab==1.0.2 +apache-airflow-providers-fab==1.0.2rc1 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1 diff --git a/constraints-3.12.txt b/constraints-3.12.txt index 7c5760af31..27b6e8c930 100644 --- a/constraints-3.12.txt +++ b/constraints-3.12.txt @@ -126,7 +126,7 @@ apache-airflow-providers-discord==3.6.0 apache-airflow-providers-docker==3.9.2 apache-airflow-providers-elasticsearch==5.3.3 apache-airflow-providers-exasol==4.4.3 -apache-airflow-providers-fab==1.0.2 +apache-airflow-providers-fab==1.0.2rc1 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1 diff --git a/constraints-3.8.txt b/constraints-3.8.txt index 22b1a7615d..e2ccd1347b 100644 --- a/constraints-3.8.txt +++ b/constraints-3.8.txt @@ -128,7 +128,7 @@ apache-airflow-providers-discord==3.6.0 apache-airflow-providers-docker==3.9.2 apache-airflow-providers-elasticsearch==5.3.3 apache-airflow-providers-exasol==4.4.3 -apache-airflow-providers-fab==1.0.2 +apache-airflow-providers-fab==1.0.2rc1 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1 diff --git a/constraints-3.9.txt b/constraints-3.9.txt index ce395911fc..307074cd76 100644 --- a/constraints-3.9.txt +++ b/constraints-3.9.txt @@ -128,7 +128,7 @@ apache-airflow-providers-discord==3.6.0 apache-airflow-providers-docker==3.9.2 apache-airflow-providers-elasticsearch==5.3.3 apache-airflow-providers-exasol==4.4.3 -apache-airflow-providers-fab==1.0.2 +apache-airflow-providers-fab==1.0.2rc1 apache-airflow-providers-facebook==3.4.0 apache-airflow-providers-ftp==3.7.0 apache-airflow-providers-github==2.5.1
(airflow) branch v2-9-test updated (3a9391c3a1 -> 50f22ffe91)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git from 3a9391c3a1 Update hatchling to latest version (1.22.5) (#38780) add 50f22ffe91 fixup! fixup! Update RELEASE_NOTES.rst No new revisions were added by this update. Summary of changes: airflow/reproducible_build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(airflow) branch v2-9-test updated (d40b751fbc -> 3a9391c3a1)
This is an automated email from the ASF dual-hosted git repository. potiuk pushed a change to branch v2-9-test in repository https://gitbox.apache.org/repos/asf/airflow.git discard d40b751fbc Update hatchling to latest version (1.22.5) (#38780) add 3a9391c3a1 Update hatchling to latest version (1.22.5) (#38780) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (d40b751fbc) \ N -- N -- N refs/heads/v2-9-test (3a9391c3a1) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes:
(airflow) branch v2-9-test updated (9930804829 -> d40b751fbc)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v2-9-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 9930804829 fixup! Update RELEASE_NOTES.rst
discard 636ad7b26d Load providers configuration when gunicorn workers start
(#38795)
add a3eb5f73d6 Load providers configuration when gunicorn workers start
(#38795)
add b38575a5a4 fixup! Update RELEASE_NOTES.rst
add 35684a82a4 Rename pre-commit scripts, drop pre_commit_ prefix (#38667)
add d40b751fbc Update hatchling to latest version (1.22.5) (#38780)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9930804829)
\
N -- N -- N refs/heads/v2-9-test (d40b751fbc)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci-image-build.yml | 2 +-
.github/workflows/prod-image-build.yml | 2 +-
.pre-commit-config.yaml| 188 ++---
README.md | 2 +-
airflow/reproducible_build.yaml| 4 +-
clients/python/pyproject.toml | 2 +-
contributing-docs/05_pull_requests.rst | 2 +-
dev/README_RELEASE_AIRFLOW.md | 4 +-
.../airflow_breeze/commands/ci_image_commands.py | 2 +-
.../src/airflow_breeze/utils/md5_build_check.py| 2 +-
docker_tests/requirements.txt | 2 +-
docs/apache-airflow/database-erd-ref.rst | 2 +-
.../installation/supported-versions.rst| 2 +-
docs/apache-airflow/migrations-ref.rst | 2 +-
pyproject.toml | 2 +-
...ments.py => base_operator_partial_arguments.py} | 0
...re_commit_boring_cyborg.py => boring_cyborg.py} | 0
...ommit_breeze_cmd_line.py => breeze_cmd_line.py} | 0
...gelog_duplicates.py => changelog_duplicates.py} | 0
...{pre_commit_chart_schema.py => chart_schema.py} | 0
...e_optional.py => check_aiobotocore_optional.py} | 0
...ate.py => check_airflow_bug_report_template.py} | 0
...s_not_used.py => check_airflow_k8s_not_used.py} | 0
...> check_cncf_k8s_used_for_k8s_executor_only.py} | 0
...ependency.py => check_common_sql_dependency.py} | 0
...able_default.py => check_deferrable_default.py} | 0
...packages_ref.py => check_extra_packages_ref.py} | 0
..._re2_imports.py => check_google_re2_imports.py} | 0
...eck_init_in_tests.py => check_init_in_tests.py} | 0
...grations_list.py => check_integrations_list.py} | 0
...check_lazy_logging.py => check_lazy_logging.py} | 0
...re_commit_check_license.py => check_license.py} | 0
..._extras.py => check_order_dockerfile_extras.py} | 0
...r_hatch_build.py => check_order_hatch_build.py} | 0
...e_commit_hooks.py => check_pre_commit_hooks.py} | 0
py => check_provider_airflow_compatibility.py} | 0
...eck_provider_docs.py => check_provider_docs.py} | 0
..._yaml_files.py => check_provider_yaml_files.py} | 0
...k_providers_init.py => check_providers_init.py} | 0
...> check_providers_subpackages_all_have_init.py} | 0
...check_system_tests.py => check_system_tests.py} | 0
...ex.py => check_system_tests_hidden_in_index.py} | 0
..._folders.py => check_tests_in_right_folders.py} | 2 +-
...o_credentials.py => checkout_no_credentials.py} | 0
...compile_www_assets.py => compile_www_assets.py} | 0
...www_assets_dev.py => compile_www_assets_dev.py} | 0
...> decorator_operator_implements_custom_name.py} | 0
...tring_param_type.py => docstring_param_type.py} | 0
...ow_diagrams.py => generate_airflow_diagrams.py} | 0
...rate_pypi_readme.py => generate_pypi_readme.py} | 0
.../{pre_commit_helm_lint.py => helm_lint.py} | 0
...ts_in_docker.py => inline_scripts_in_docker.py} | 0
...re_commit_insert_extras.py => insert_extras.py} | 0
.../{pre_commit_json_schema.py => json_schema.py} | 0
.../{pre_commit_kubeconform.py => kubeconform.py} | 0
...ommit_lint_dockerfile.py => lint_dockerfile.py} | 0
...mit_local_yml_mounts.py => local_yml_mounts.py} | 0
...gration_reference.py => migration_reference.py} | 0
.../ci/pre_commit/{pre_
