justinpakzad commented on code in PR #70446:
URL: https://github.com/apache/airflow/pull/70446#discussion_r4115803217
##########
airflow-core/src/airflow/cli/commands/dag_command.py:
##########
@@ -80,6 +82,95 @@
_RUN_CHUNK_SIZE = 500
+def _normalize_serialized_dag_for_stability_check(serialized_dag: dict[str,
Any]) -> dict[str, Any]:
+ normalized = SerializedDagModel._sort_serialized_dag_dict(serialized_dag)
+ normalized["dag"].pop("fileloc", None)
+ normalized["dag"].pop("bundle_name", None)
+ return normalized
+
+
+def _serialize_dag_for_stability_check(dag: DAG) -> tuple[str, dict[str, Any]]:
+ serialized_dag = DagSerialization.to_dict(dag)
+ return SerializedDagModel.hash(serialized_dag),
_normalize_serialized_dag_for_stability_check(
+ serialized_dag
+ )
+
+
+def _format_stability_diff(
+ dag_id: str,
+ first_serialized_dag: dict[str, Any],
+ second_serialized_dag: dict[str, Any],
+) -> str:
+ before = json.dumps(first_serialized_dag, indent=2,
sort_keys=True).splitlines()
+ after = json.dumps(second_serialized_dag, indent=2,
sort_keys=True).splitlines()
+ diff = difflib.unified_diff(
+ before,
+ after,
+ fromfile=f"parse 1: {dag_id} ",
+ tofile=f"parse 2: {dag_id} ",
+ lineterm="",
+ )
+ return "\n".join(diff)
+
+
+def _parse_dags_for_stability_check(dag_folder: str | None) -> DagBag:
+ return DagBag(dag_folder=dag_folder, load_op_links=False)
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+def dag_stability_check(args) -> None:
+ dag_hashes_by_id: dict[str, list[str]] = {}
+ serialized_dags_by_id: dict[str, list[dict[str, Any]]] = {}
+ seen_dag_ids: set[str] = set()
+ nParse = (
+ 2 # NOTE: Set parsing number 2, in most of the case twice parse
should catch the stability issues.
Review Comment:
On second thought, if we did make this configurable, we would need to figure
out which diffs to show since there could be many different versions. So maybe
keeping this fixed at 2 is actually a better approach for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
