jedcunningham commented on code in PR #39138:
URL: https://github.com/apache/airflow/pull/39138#discussion_r1589372810
##########
airflow/api_connexion/openapi/v1.yaml:
##########
@@ -1091,6 +1091,29 @@ paths:
"404":
$ref: "#/components/responses/NotFound"
+ /dag/parse/{file_token}:
Review Comment:
```suggestion
/parseDagFie/{file_token}:
```
Or something similar. Note the dag source endpoint is `/dagSources`.
##########
airflow/api_connexion/openapi/v1.yaml:
##########
@@ -1091,6 +1091,29 @@ paths:
"404":
$ref: "#/components/responses/NotFound"
+ /dag/parse/{file_token}:
+ parameters:
+ - $ref: "#/components/parameters/FileToken"
+
+ put:
+ summary: Request re-parsing of DAGs
Review Comment:
```suggestion
summary: Request re-parsing of a DAG file
```
##########
airflow/api_connexion/endpoints/dag_parsing.py:
##########
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Sequence
+
+from flask import Response, current_app
+from itsdangerous import BadSignature, URLSafeSerializer
+from sqlalchemy import exc
+
+from airflow.api_connexion import security
+from airflow.api_connexion.exceptions import NotFound, PermissionDenied
+from airflow.auth.managers.models.resource_details import DagDetails
+from airflow.models.dag import DagModel
+from airflow.models.dagbag import DagPriorityParsingRequest
+from airflow.utils.session import NEW_SESSION, provide_session
+from airflow.www.extensions.init_auth_manager import get_auth_manager
+
+if TYPE_CHECKING:
+ from sqlalchemy.orm import Session
+
+ from airflow.auth.managers.models.batch_apis import IsAuthorizedDagRequest
+
+
[email protected]_access_dag("PUT")
+@provide_session
+def reparse_dags(*, file_token: str, session: Session = NEW_SESSION) ->
Response:
+ """Request re-parsing DAGs."""
Review Comment:
```suggestion
"""Request re-parsing a DAG file."""
```
##########
airflow/api_connexion/endpoints/dag_parsing.py:
##########
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Sequence
+
+from flask import Response, current_app
+from itsdangerous import BadSignature, URLSafeSerializer
+from sqlalchemy import exc
+
+from airflow.api_connexion import security
+from airflow.api_connexion.exceptions import NotFound, PermissionDenied
+from airflow.auth.managers.models.resource_details import DagDetails
+from airflow.models.dag import DagModel
+from airflow.models.dagbag import DagPriorityParsingRequest
+from airflow.utils.session import NEW_SESSION, provide_session
+from airflow.www.extensions.init_auth_manager import get_auth_manager
+
+if TYPE_CHECKING:
+ from sqlalchemy.orm import Session
+
+ from airflow.auth.managers.models.batch_apis import IsAuthorizedDagRequest
+
+
[email protected]_access_dag("PUT")
+@provide_session
+def reparse_dags(*, file_token: str, session: Session = NEW_SESSION) ->
Response:
+ """Request re-parsing DAGs."""
+ secret_key = current_app.config["SECRET_KEY"]
+ auth_s = URLSafeSerializer(secret_key)
+ try:
+ path = auth_s.loads(file_token)
+ dag_ids = session.query(DagModel.dag_id).filter(DagModel.fileloc ==
path).all()
+ if len(dag_ids) == 0:
+ raise FileNotFoundError
+ except (BadSignature, FileNotFoundError):
+ raise NotFound("File not found")
+
+ requests: Sequence[IsAuthorizedDagRequest] = [
+ {
+ "method": "PUT",
+ "details": DagDetails(id=dag_id[0]),
+ }
+ for dag_id in dag_ids
+ ]
+ # Check if user has read access to all the DAGs defined in the file
+ if not get_auth_manager().batch_is_authorized_dag(requests):
+ raise PermissionDenied()
+
+ parsing_request = DagPriorityParsingRequest(fileloc=path)
+ session.add(parsing_request)
+ try:
+ session.commit()
+ except exc.IntegrityError:
+ session.rollback()
+ return Response("Duplicate request", HTTPStatus.CONFLICT)
Review Comment:
I feel like this shouldn't be a client error. Either 201 or 204?
##########
airflow/api_connexion/openapi/v1.yaml:
##########
@@ -1091,6 +1091,29 @@ paths:
"404":
$ref: "#/components/responses/NotFound"
+ /dag/parse/{file_token}:
+ parameters:
+ - $ref: "#/components/parameters/FileToken"
+
+ put:
+ summary: Request re-parsing of DAGs
+ description: >
+ Request re-parsing of existing DAGs
Review Comment:
```suggestion
Request re-parsing of existing DAG files using a file token.
```
##########
airflow/api_connexion/endpoints/dag_parsing.py:
##########
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Sequence
+
+from flask import Response, current_app
+from itsdangerous import BadSignature, URLSafeSerializer
+from sqlalchemy import exc
+
+from airflow.api_connexion import security
+from airflow.api_connexion.exceptions import NotFound, PermissionDenied
+from airflow.auth.managers.models.resource_details import DagDetails
+from airflow.models.dag import DagModel
+from airflow.models.dagbag import DagPriorityParsingRequest
+from airflow.utils.session import NEW_SESSION, provide_session
+from airflow.www.extensions.init_auth_manager import get_auth_manager
+
+if TYPE_CHECKING:
+ from sqlalchemy.orm import Session
+
+ from airflow.auth.managers.models.batch_apis import IsAuthorizedDagRequest
+
+
[email protected]_access_dag("PUT")
+@provide_session
+def reparse_dags(*, file_token: str, session: Session = NEW_SESSION) ->
Response:
Review Comment:
```suggestion
def reparse_dag_file(*, file_token: str, session: Session = NEW_SESSION) ->
Response:
```
##########
airflow/models/dagbag.py:
##########
@@ -727,3 +734,43 @@ def _sync_perm_for_dag(cls, dag: DAG, session: Session =
NEW_SESSION):
security_manager = ApplessAirflowSecurityManager(session=session)
security_manager.sync_perm_for_dag(root_dag_id, dag.access_control)
+
+
+def generate_md5_hash(context):
+ fileloc = context.get_current_parameters()["fileloc"]
+ return hashlib.md5(fileloc.encode()).hexdigest()
+
+
+class DagPriorityParsingRequest(Base):
+ """Model to store the dag parsing requests that will be prioritized when
parsing files."""
+
+ __tablename__ = "dag_priority_parsing_request"
+
+ # Adding a unique constraint to fileloc results in the creation of an
index and we have a limitation
+ # on the size of the string we can use in the index for MySql DB. We also
have to keep the fileloc
Review Comment:
```suggestion
# on the size of the string we can use in the index for MySQL DB. We
also have to keep the fileloc
```
nit
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
