Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2130811150 @ashb thanks for the review. I share your concern about resource consumption issues when loading all dags from the DB. I tried to query the JSON via many ways. One would be: `session.query(SerializedDagModel).filter(SerializedDagModel.__table__.columns.data['dag_id'].astext == dag_id)` yet I always end up in `NotImplementedError: Operator 'getitem' is not supported on this expression`. Can somebody support me here with a minimal working code snippet? In the repo, I cannot find any comparable code that queries the serialized dags in the DB. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
jscheffl commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1581455829
##
airflow/www/views.py:
##
@@ -5415,51 +5453,78 @@ class AutocompleteView(AirflowBaseView):
def autocomplete(self, session: Session = NEW_SESSION):
"""Autocomplete."""
query = unquote(request.args.get("query", ""))
+query_prefix = ""
+prefix_search_match = re2.match(r"(?i)(dag|owner|task):\s*", query)
+if prefix_search_match:
+query_prefix = prefix_search_match[0].lower()
+query = query[len(query_prefix) :]
if not query:
return flask.json.jsonify([])
-# Provide suggestions of dag_ids and owners
-dag_ids_query = select(
-sqla.literal("dag").label("type"),
-DagModel.dag_id.label("name"),
-DagModel._dag_display_property_value.label("dag_display_name"),
-).where(
-~DagModel.is_subdag,
-DagModel.is_active,
-or_(
-DagModel.dag_id.ilike(f"%{query}%"),
-DagModel._dag_display_property_value.ilike(f"%{query}%"),
-),
-)
+status = flask_session.get(FILTER_STATUS_COOKIE)
+filter_dag_ids = get_auth_manager().get_permitted_dag_ids(user=g.user)
-owners_query = (
-select(
-sqla.literal("owner").label("type"),
-DagModel.owners.label("name"),
-sqla.literal(None).label("dag_display_name"),
+def _filter_dags_query(dags_query: Select) -> Select:
+# Hide DAGs if not showing status: "all"
+if status == "active":
+dags_query = dags_query.where(~DagModel.is_paused)
+elif status == "paused":
+dags_query = dags_query.where(DagModel.is_paused)
+return dags_query.where(
+~DagModel.is_subdag, DagModel.is_active,
DagModel.dag_id.in_(filter_dag_ids)
)
-.distinct()
-.where(~DagModel.is_subdag, DagModel.is_active,
DagModel.owners.ilike(f"%{query}%"))
-)
-# Hide DAGs if not showing status: "all"
-status = flask_session.get(FILTER_STATUS_COOKIE)
-if status == "active":
-dag_ids_query = dag_ids_query.where(~DagModel.is_paused)
-owners_query = owners_query.where(~DagModel.is_paused)
-elif status == "paused":
-dag_ids_query = dag_ids_query.where(DagModel.is_paused)
-owners_query = owners_query.where(DagModel.is_paused)
+if query_prefix.startswith("task:"):
+# Provide suggestions of task_ids
+dags_query = _filter_dags_query(dags_query=select(DagModel.dag_id))
+filtered_dag_ids = session.scalars(dags_query).all()
+dag_bag = get_airflow_app().dag_bag
+filtered_dags = [dag_bag.get_dag(dag_id, session=session) for
dag_id in filtered_dag_ids]
Review Comment:
Do you have specific DAGs loaded? Which chars are you entering? Can you post
the stack trace?
I tested locally on my side and for me it looks good.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1581435304
##
airflow/www/views.py:
##
@@ -5415,51 +5453,78 @@ class AutocompleteView(AirflowBaseView):
def autocomplete(self, session: Session = NEW_SESSION):
"""Autocomplete."""
query = unquote(request.args.get("query", ""))
+query_prefix = ""
+prefix_search_match = re2.match(r"(?i)(dag|owner|task):\s*", query)
+if prefix_search_match:
+query_prefix = prefix_search_match[0].lower()
+query = query[len(query_prefix) :]
if not query:
return flask.json.jsonify([])
-# Provide suggestions of dag_ids and owners
-dag_ids_query = select(
-sqla.literal("dag").label("type"),
-DagModel.dag_id.label("name"),
-DagModel._dag_display_property_value.label("dag_display_name"),
-).where(
-~DagModel.is_subdag,
-DagModel.is_active,
-or_(
-DagModel.dag_id.ilike(f"%{query}%"),
-DagModel._dag_display_property_value.ilike(f"%{query}%"),
-),
-)
+status = flask_session.get(FILTER_STATUS_COOKIE)
+filter_dag_ids = get_auth_manager().get_permitted_dag_ids(user=g.user)
-owners_query = (
-select(
-sqla.literal("owner").label("type"),
-DagModel.owners.label("name"),
-sqla.literal(None).label("dag_display_name"),
+def _filter_dags_query(dags_query: Select) -> Select:
+# Hide DAGs if not showing status: "all"
+if status == "active":
+dags_query = dags_query.where(~DagModel.is_paused)
+elif status == "paused":
+dags_query = dags_query.where(DagModel.is_paused)
+return dags_query.where(
+~DagModel.is_subdag, DagModel.is_active,
DagModel.dag_id.in_(filter_dag_ids)
)
-.distinct()
-.where(~DagModel.is_subdag, DagModel.is_active,
DagModel.owners.ilike(f"%{query}%"))
-)
-# Hide DAGs if not showing status: "all"
-status = flask_session.get(FILTER_STATUS_COOKIE)
-if status == "active":
-dag_ids_query = dag_ids_query.where(~DagModel.is_paused)
-owners_query = owners_query.where(~DagModel.is_paused)
-elif status == "paused":
-dag_ids_query = dag_ids_query.where(DagModel.is_paused)
-owners_query = owners_query.where(DagModel.is_paused)
+if query_prefix.startswith("task:"):
+# Provide suggestions of task_ids
+dags_query = _filter_dags_query(dags_query=select(DagModel.dag_id))
+filtered_dag_ids = session.scalars(dags_query).all()
+dag_bag = get_airflow_app().dag_bag
+filtered_dags = [dag_bag.get_dag(dag_id, session=session) for
dag_id in filtered_dag_ids]
Review Comment:
In testing locally, I am keep getting errors from these dag_bag fetches:
```ValueError("Invalid value for `name`, must not be `None`")```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2079612257 > Do we want to include searching by task_display_name in this PR or in a follow up? Searching by task_display_name is already included and the PR does not change this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2076827199 > Let's rebase. Quite some effort this time. Included the display_dag_name logic -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
jscheffl commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2067295317 @bbovenzi or @eladkal Can yo umake a 2nd pass review, then I thing we should merge this. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2041956580 > Thanks for the fix, I can confirm it works now as advertised. > > I approve from my side. I just kindly request to also alter the documentation a bit. Else the risk is high that nobody will be able to find and discover this cool freature - especially searching for tasks. Can you add / adjust documentation in `docs/apache-airflow/ui.rst` Thanks a lot! I now also extended the UI docs. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2041368166 @jscheffl > I checked and I have no auto-complete and suggestions when I type as described in the PR descriptions. Tested with Firefox and Chromium. Did I mis-undertstand the PR that suggestions are not there? Or are the suggestions browser specific? A bug was introduced when rebasing, which caused the autocomplete to fail - see the FIX commit. The feature works for me now in both Firefox and Chromium. > Also when not prefixing `dag: ` in the search the text entered always filters for the DAG ID. Search for tasks and owner though is working. But `dag: ` prefix smells un-needed if this is the default. Or shall the default be to make a global search? The default is to search by both dag id and owner. The `dag: ` prefix resolves confusion in case a dag id and owner are named the same. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
jscheffl commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2030075088 I checked and I have no auto-complete and suggestions when I type as described in the PR descriptions. Tested with Firefox and Chromium. Did I mis-undertstand the PR that suggestions are not there? Or are the suggestions browser specific? Also when not prefixing `dag: ` in the search the text entered always filters for the DAG ID. Search for tasks and owner though is working. But `dag: ` prefix smells un-needed if this is the default. Or shall the default be to make a global search? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
eladkal commented on PR #37436: URL: https://github.com/apache/airflow/pull/37436#issuecomment-2029323779 @pruoff can you rebase and resolve conflicts? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1496808744
##
airflow/www/static/js/dags.js:
##
@@ -126,16 +126,75 @@ $(".typeahead").typeahead({
success: callback,
});
},
+ matcher(item) {
+const it = this.displayText(item);
+const query = this.query.toLowerCase();
+const queryValue = query.replace(/^([a-z]+:\s*)/i, "");
+
+return it.toLowerCase().indexOf(queryValue) !== -1;
+ },
+ displayText(item) {
+if (typeof item !== "undefined" && typeof item.name !== "undefined") {
+ if (item.type === "task") {
+return `${item.type}: ${item.name} in ${item.dag_id}`;
+ }
+ return `${item.type}: ${item.name}`;
+}
+return item;
+ },
+ sorter(items) {
+const beginswith = [];
+const caseSensitive = [];
+const caseInsensitive = [];
+let item;
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+while (items.length) {
+ item = items.shift();
+ if (!item.name.toLowerCase().indexOf(query.toLowerCase())) {
+beginswith.push(item);
+ } else if (item.name.indexOf(query) !== -1) {
+caseSensitive.push(item);
+ } else {
+caseInsensitive.push(item);
+ }
+}
+
+return beginswith.concat(caseSensitive, caseInsensitive);
+ },
+ highlighter(displayText, item) {
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+if (query === "") {
+ return displayText;
+}
+
+let { name } = item;
+query = query.replace(/[()/.*+?[\]]/g, (mat) => `\\${mat}`);
+const reg = new RegExp(query, "gi");
+name = name.replace(name, name.replace(reg, "$&"));
+return displayText.replace(item.name, name);
+ },
autoSelect: false,
afterSelect(value) {
const query = new URLSearchParams(window.location.search);
-query.set("search", value.name);
if (value.type === "owner") {
+ query.set("search", value.name);
window.location = `${DAGS_INDEX}?${query}`;
}
if (value.type === "dag") {
+ query.set("search", value.name);
window.location = `${gridUrl.replace("__DAG_ID__",
value.name)}?${query}`;
}
+if (value.type === "task") {
+ query.set("search", value.dag_id);
Review Comment:
```suggestion
query.set("task_id", value.name);
```
Rebase and then changing the search param will allow you to navigate
directly to the task in question
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1496812864
##
airflow/www/static/js/dags.js:
##
@@ -126,16 +126,75 @@ $(".typeahead").typeahead({
success: callback,
});
},
+ matcher(item) {
+const it = this.displayText(item);
+const query = this.query.toLowerCase();
+const queryValue = query.replace(/^([a-z]+:\s*)/i, "");
+
+return it.toLowerCase().indexOf(queryValue) !== -1;
+ },
+ displayText(item) {
+if (typeof item !== "undefined" && typeof item.name !== "undefined") {
+ if (item.type === "task") {
+return `${item.type}: ${item.name} in ${item.dag_id}`;
+ }
+ return `${item.type}: ${item.name}`;
+}
+return item;
+ },
+ sorter(items) {
+const beginswith = [];
+const caseSensitive = [];
+const caseInsensitive = [];
+let item;
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
Review Comment:
Let's create a new variable for query instead of using `let` and reassigning
its value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1496812008
##
airflow/www/static/js/dags.js:
##
@@ -126,16 +126,75 @@ $(".typeahead").typeahead({
success: callback,
});
},
+ matcher(item) {
+const it = this.displayText(item);
+const query = this.query.toLowerCase();
+const queryValue = query.replace(/^([a-z]+:\s*)/i, "");
+
+return it.toLowerCase().indexOf(queryValue) !== -1;
+ },
+ displayText(item) {
+if (typeof item !== "undefined" && typeof item.name !== "undefined") {
Review Comment:
Do we want to check against the string `"undefined"` or the javascript value
`undefined`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1496808744
##
airflow/www/static/js/dags.js:
##
@@ -126,16 +126,75 @@ $(".typeahead").typeahead({
success: callback,
});
},
+ matcher(item) {
+const it = this.displayText(item);
+const query = this.query.toLowerCase();
+const queryValue = query.replace(/^([a-z]+:\s*)/i, "");
+
+return it.toLowerCase().indexOf(queryValue) !== -1;
+ },
+ displayText(item) {
+if (typeof item !== "undefined" && typeof item.name !== "undefined") {
+ if (item.type === "task") {
+return `${item.type}: ${item.name} in ${item.dag_id}`;
+ }
+ return `${item.type}: ${item.name}`;
+}
+return item;
+ },
+ sorter(items) {
+const beginswith = [];
+const caseSensitive = [];
+const caseInsensitive = [];
+let item;
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+while (items.length) {
+ item = items.shift();
+ if (!item.name.toLowerCase().indexOf(query.toLowerCase())) {
+beginswith.push(item);
+ } else if (item.name.indexOf(query) !== -1) {
+caseSensitive.push(item);
+ } else {
+caseInsensitive.push(item);
+ }
+}
+
+return beginswith.concat(caseSensitive, caseInsensitive);
+ },
+ highlighter(displayText, item) {
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+if (query === "") {
+ return displayText;
+}
+
+let { name } = item;
+query = query.replace(/[()/.*+?[\]]/g, (mat) => `\\${mat}`);
+const reg = new RegExp(query, "gi");
+name = name.replace(name, name.replace(reg, "$&"));
+return displayText.replace(item.name, name);
+ },
autoSelect: false,
afterSelect(value) {
const query = new URLSearchParams(window.location.search);
-query.set("search", value.name);
if (value.type === "owner") {
+ query.set("search", value.name);
window.location = `${DAGS_INDEX}?${query}`;
}
if (value.type === "dag") {
+ query.set("search", value.name);
window.location = `${gridUrl.replace("__DAG_ID__",
value.name)}?${query}`;
}
+if (value.type === "task") {
+ query.set("search", value.dag_id);
Review Comment:
```suggestion
query.set("task_id", value.name);
```
Rebase and then changing the searh param will allow you to navigate directly
to the task in question
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
Re: [PR] Added Feature: search dags by task id with suggestions [airflow]
bbovenzi commented on code in PR #37436:
URL: https://github.com/apache/airflow/pull/37436#discussion_r1496809388
##
airflow/www/static/js/dags.js:
##
@@ -126,16 +126,75 @@ $(".typeahead").typeahead({
success: callback,
});
},
+ matcher(item) {
+const it = this.displayText(item);
+const query = this.query.toLowerCase();
+const queryValue = query.replace(/^([a-z]+:\s*)/i, "");
+
+return it.toLowerCase().indexOf(queryValue) !== -1;
+ },
+ displayText(item) {
+if (typeof item !== "undefined" && typeof item.name !== "undefined") {
+ if (item.type === "task") {
+return `${item.type}: ${item.name} in ${item.dag_id}`;
+ }
+ return `${item.type}: ${item.name}`;
+}
+return item;
+ },
+ sorter(items) {
+const beginswith = [];
+const caseSensitive = [];
+const caseInsensitive = [];
+let item;
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+while (items.length) {
+ item = items.shift();
+ if (!item.name.toLowerCase().indexOf(query.toLowerCase())) {
+beginswith.push(item);
+ } else if (item.name.indexOf(query) !== -1) {
+caseSensitive.push(item);
+ } else {
+caseInsensitive.push(item);
+ }
+}
+
+return beginswith.concat(caseSensitive, caseInsensitive);
+ },
+ highlighter(displayText, item) {
+let { query } = this;
+query = query.replace(/^([a-z]+:\s*)/i, "");
+
+if (query === "") {
+ return displayText;
+}
+
+let { name } = item;
+query = query.replace(/[()/.*+?[\]]/g, (mat) => `\\${mat}`);
+const reg = new RegExp(query, "gi");
+name = name.replace(name, name.replace(reg, "$&"));
+return displayText.replace(item.name, name);
+ },
autoSelect: false,
afterSelect(value) {
const query = new URLSearchParams(window.location.search);
-query.set("search", value.name);
if (value.type === "owner") {
+ query.set("search", value.name);
window.location = `${DAGS_INDEX}?${query}`;
}
if (value.type === "dag") {
+ query.set("search", value.name);
Review Comment:
We don't want the query for the gridUrl since it's already directly in the
url
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[PR] Added Feature: search dags by task id with suggestions [airflow]
pruoff opened a new pull request, #37436: URL: https://github.com/apache/airflow/pull/37436 This PR closes: #37106 Previously the DAG search bar could be used to search by DAG id and owner. Changes in /home UI from user's perspective: * DAG search with prefix "task: " filters all DAGs by case-insensitive pattern-matching of the provided task_id. * suggestions of task_ids are automatically shown when typing. Since the same task_id can be present in many dags, an italic " in " is shown following the task_id suggestion. * actively limit the DAG search scope by prefixing "dag:" for dag_id search and "owner:" for owner search. * autocompleted suggestions are prefixed by their type e.g. "owner: airflow"  -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@airflow.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
