potiuk commented on code in PR #59183: URL: https://github.com/apache/airflow/pull/59183#discussion_r2608552127
########## airflow-core/src/airflow/assets/manager.py: ########## @@ -54,6 +55,59 @@ log = structlog.get_logger(__name__) +@contextmanager +def _aquire_apdr_lock( + *, + session: Session, + dag_id: str, + partition_key: str, + asset_id: int, + max_retries: int = 10, + retry_delay: float = 0.05, +): + """ + Context manager to acquire a lock for AssetPartitionDagRun creation. + + - SQLite: uses AssetPartitionDagRunMutexLock table as row-level lock is not supported + - Postgres/MySQL: uses row-level lock on AssetModel. + """ + if get_dialect_name(session) == "sqlite": + from airflow.models.asset import AssetPartitionDagRunMutexLock + + for _ in range(max_retries): + try: + mutex = AssetPartitionDagRunMutexLock(target_dag_id=dag_id, partition_key=partition_key) + session.add(mutex) + session.flush() + try: + yield # mutex acquired + finally: + session.delete(mutex) Review Comment: But also I am not sure if there can be two ADPR records added in parallel in sqlite. Do we even support multiple schedulers at the same time in sqlite? cc @dstandish -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
