jason810496 commented on code in PR #68082:
URL: https://github.com/apache/airflow/pull/68082#discussion_r3566041501
##########
providers/apache/kafka/src/airflow/providers/apache/kafka/hooks/base.py:
##########
@@ -85,10 +109,38 @@ def get_ui_field_behaviour(cls) -> dict[str, Any]:
def _get_client(self, config) -> Any:
return AdminClient(config)
+ def _get_config(self) -> dict[str, Any]:
+ """Resolve the client config: the connection extras, with
``config_dict`` keys taking precedence."""
+ config_dict = self.config_dict or {}
+ try:
+ config =
dict(self.get_connection(self.kafka_config_id).extra_dejson)
+ except Exception:
+ # The connection lookup can fail for different reasons depending
on the
+ # component the hook runs in (missing connection, no DB/API
access). When
+ # ``config_dict`` carries the brokers the hook can run without a
connection;
+ # otherwise re-raise.
+ if not config_dict.get("bootstrap.servers"):
+ raise
+ self.log.debug(
+ "Kafka connection %r is not available; building the client
from 'config_dict' only.",
+ self.kafka_config_id,
+ )
+ config = {}
+ config.update(config_dict)
+ self._resolve_callbacks(config)
Review Comment:
From my perspective, we could still resolve the importable callback from the
connection extra instead of having a new `config_dict` for all the Kafka Hooks.
WDYT?
##########
providers/apache/kafka/src/airflow/providers/apache/kafka/plugins/listener.py:
##########
@@ -0,0 +1,475 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import atexit
+import json
+import logging
+import os
+import time
+from datetime import datetime, timezone
+from fnmatch import fnmatch
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any
+
+from airflow.providers.apache.kafka.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.providers.common.compat.sdk import AirflowPlugin, conf, hookimpl
+from airflow.utils.net import get_hostname
+
+if TYPE_CHECKING:
+ from confluent_kafka import Producer
+ from sqlalchemy.orm import Session
+
+ from airflow.models import DagRun, TaskInstance
+ from airflow.sdk.execution_time.task_runner import RuntimeTaskInstance
+ from airflow.utils.state import TaskInstanceState
+
+log = logging.getLogger(__name__)
+
+CONFIG_SECTION = "kafka_listener"
+SCHEMA_VERSION = 1
+
+
+@lru_cache(maxsize=1)
+def _dag_run_events_enabled() -> bool:
+ return conf.getboolean(CONFIG_SECTION, "dag_run_events_enabled",
fallback="False")
+
+
+@lru_cache(maxsize=1)
+def _task_instance_events_enabled() -> bool:
+ return conf.getboolean(CONFIG_SECTION, "task_instance_events_enabled",
fallback="False")
+
+
+@lru_cache(maxsize=1)
+def _get_kafka_config_id() -> str:
+ return conf.get(CONFIG_SECTION, "kafka_config_id", fallback="").strip()
+
+
+@lru_cache(maxsize=1)
+def _get_config_dict() -> dict[str, Any] | None:
+ raw_dict_str = conf.get(CONFIG_SECTION, "config_dict", fallback="").strip()
+ if not raw_dict_str:
+ return None
+ config_dict = json.loads(raw_dict_str)
+ if not isinstance(config_dict, dict):
+ raise ValueError(
+ f"The 'config_dict' option of the [{CONFIG_SECTION}] config
section "
+ f"must be a JSON object, got: {type(config_dict).__name__}"
+ )
+ return config_dict
+
+
+@lru_cache(maxsize=1)
+def _get_topic() -> str:
+ return conf.get(CONFIG_SECTION, "topic", fallback="airflow.events")
+
+
+@lru_cache(maxsize=1)
+def _get_source() -> str:
+ source_from_conf = conf.get(CONFIG_SECTION, "source", fallback="")
+ # Default to the current hostname, if un-set.
+ return source_from_conf or get_hostname()
+
+
+def _parse_filter_patterns_to_tuple(raw: str) -> tuple[str, ...]:
+ return tuple(p.strip() for p in raw.split(",") if p.strip())
+
+
+@lru_cache(maxsize=1)
+def _get_dag_run_dag_id_allowlist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(conf.get(CONFIG_SECTION,
"dag_run_dag_id_allowlist", fallback=""))
+
+
+@lru_cache(maxsize=1)
+def _get_dag_run_dag_id_denylist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(conf.get(CONFIG_SECTION,
"dag_run_dag_id_denylist", fallback=""))
+
+
+@lru_cache(maxsize=1)
+def _get_task_instance_dag_id_allowlist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(
+ conf.get(CONFIG_SECTION, "task_instance_dag_id_allowlist", fallback="")
+ )
+
+
+@lru_cache(maxsize=1)
+def _get_task_instance_dag_id_denylist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(
+ conf.get(CONFIG_SECTION, "task_instance_dag_id_denylist", fallback="")
+ )
+
+
+@lru_cache(maxsize=1)
+def _get_task_instance_task_id_allowlist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(
+ conf.get(CONFIG_SECTION, "task_instance_task_id_allowlist",
fallback="")
+ )
+
+
+@lru_cache(maxsize=1)
+def _get_task_instance_task_id_denylist() -> tuple[str, ...]:
+ return _parse_filter_patterns_to_tuple(
+ conf.get(CONFIG_SECTION, "task_instance_task_id_denylist", fallback="")
+ )
+
+
+@lru_cache(maxsize=1)
+def _get_topic_check_timeout() -> int:
+ return conf.getint(CONFIG_SECTION, "topic_check_timeout", fallback="10")
+
+
+@lru_cache(maxsize=1)
+def _get_topic_check_retry_interval() -> int:
+ return conf.getint(CONFIG_SECTION, "topic_check_retry_interval",
fallback="60")
+
+
+def _id_is_allowed(id_to_check: str, allowlist: tuple[str, ...], denylist:
tuple[str, ...]) -> bool:
+ """Deny takes precedence; empty allowlist means 'allow all'."""
+ if denylist and any(fnmatch(id_to_check, id_pattern) for id_pattern in
denylist):
+ return False
+ if allowlist and not any(fnmatch(id_to_check, id_pattern) for id_pattern
in allowlist):
+ return False
+ return True
+
+
+def _dag_run_event_allowed(dag_id: str) -> bool:
+ return _id_is_allowed(
+ dag_id,
+ _get_dag_run_dag_id_allowlist(),
+ _get_dag_run_dag_id_denylist(),
+ )
+
+
+def _task_instance_event_allowed(dag_id: str, task_id: str) -> bool:
+ return _id_is_allowed(
+ dag_id,
+ _get_task_instance_dag_id_allowlist(),
+ _get_task_instance_dag_id_denylist(),
+ ) and _id_is_allowed(
+ task_id,
+ _get_task_instance_task_id_allowlist(),
+ _get_task_instance_task_id_denylist(),
+ )
+
+
+# Listener-owned producer and topic state for the lifetime of the process.
+# The producer is built once via KafkaProducerHook and kept for the process
lifetime;
+# the topic flags track whether the topic exists on the broker and whether
we're
+# currently in a back-off window after a failed topic check.
+_producer: Producer | None = None
+_topic_exists: bool = False
Review Comment:
Do we need to reset the `_check_topic_exists` flag if we really encounter
the topic not found (e.g. someone accidental delete the topic)?
Perhaps as follow-up, non-blocking atm, since user should be able to tell
from the error loggings.
##########
providers/apache/kafka/src/airflow/providers/apache/kafka/hooks/base.py:
##########
@@ -58,18 +64,36 @@ class KafkaBaseHook(BaseHook):
"""
A base hook for interacting with Apache Kafka.
+ The Airflow connection's extra JSON is merged with the ``config_dict``.
+ Each ``config_dict`` key overrides the same key from the connection.
+ When the connection is not available and ``config_dict`` contains the key
+ ``bootstrap.servers``, the client is built from ``config_dict`` alone
+ without requiring a connection.
+
:param kafka_config_id: The connection object to use, defaults to
"kafka_default"
+ :param config_dict: Optional confluent-kafka client configuration
+ (e.g. ``{"bootstrap.servers": "localhost:9092"}``). Each key overrides
the same
+ key from the connection's config. Callback options (``error_cb``,
``throttle_cb``,
+ ``stats_cb``, ``log_cb``, ``oauth_cb``, ``on_commit``) may be
callables or
+ dotted-path strings, which are resolved before the client is built.
"""
conn_name_attr = "kafka_config_id"
default_conn_name = "kafka_default"
conn_type = "kafka"
hook_name = "Apache Kafka"
- def __init__(self, kafka_config_id=default_conn_name, *args, **kwargs):
+ def __init__(
+ self,
+ kafka_config_id=default_conn_name,
+ config_dict: dict[str, Any] | None = None,
Review Comment:
I'm still a bit concern about exposing the `config_dict` to bypass the
connection.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
