kaxil commented on code in PR #70498: URL: https://github.com/apache/airflow/pull/70498#discussion_r3668354854
########## registry/src/_data/providerKeywordMatch.js: ########## @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Shared by providerCategoryMap.js and exploreCategoryProviders.js so the +// two directions of the same provider <-> explore-category matching (used +// respectively by the per-provider page's category chips and the Explore +// landing page's per-category provider listing) can't drift apart. + +// Two arbitrary strings "match" a keyword if either contains the other +// (case-insensitively), mirroring the original id-only substring check. +function fuzzyIncludes(value, keyword) { + if (!value) { + return false; + } + const normalizedValue = value.toLowerCase(); + const normalizedKeyword = keyword.toLowerCase(); + return normalizedValue.includes(normalizedKeyword) || normalizedKeyword.includes(normalizedValue); +} + +// Every string a provider is searchable by: its id/slug, its declared +// integration names (provider.categories[].name — e.g. "LangChain", +// "Pydantic AI"), and, once populated, each connection type's declared +// external integrations. The latter field doesn't exist in the generated +// data yet, so this reads as an empty list until a future change in +// dev/registry/extract_metadata.py starts populating it — no further +// changes needed here when that lands. +function collectSearchableValues(provider) { + const values = [provider.id]; + for (const category of provider.categories || []) { + if (category.name) { + values.push(category.name); + } + } + for (const connectionType of provider.connection_types || []) { + for (const externalIntegration of connectionType.external_integrations || []) { Review Comment: This can't ever be populated as written. `ConnectionTypeContract` in `dev/registry/registry_contract_models.py` sets `extra="forbid"` and declares only `conn_type`, `hook_class`, `docs_url`, and `extract_metadata.py` validates providers.json against it before writing. Adding the field upstream fails validation first, so the contract change and these four lines would land together anyway. I'd drop the loop and the comment above it for now. ########## registry/src/_data/exploreCategories.js: ########## @@ -31,31 +31,31 @@ module.exports = [ name: 'Databases', icon: '<svg fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 7v10c0 2.21 3.582 4 8 4s8-1.79 8-4V7M4 7c0 2.21 3.582 4 8 4s8-1.79 8-4M4 7c0-2.21 3.582-4 8-4s8 1.79 8 4" /></svg>', color: 'blue', - keywords: ['postgres', 'mysql', 'mongo', 'redis', 'neo4j', 'elasticsearch', 'cassandra', 'couchbase', 'influxdb', 'sqlite', 'odbc', 'jdbc', 'common-sql'], + keywords: ['postgres', 'mysql', 'mongo', 'redis', 'neo4j', 'elasticsearch', 'cassandra', 'couchbase', 'influxdb', 'sqlite', 'odbc', 'jdbc', 'common-sql', 'mssql', 'opensearch', 'cloudant', 'arangodb'], description: 'SQL, NoSQL, and time-series databases', }, { id: 'data-warehouses', name: 'Data Warehouses', icon: '<svg fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 11H5m14 0a2 2 0 012 2v6a2 2 0 01-2 2H5a2 2 0 01-2-2v-6a2 2 0 012-2m14 0V9a2 2 0 00-2-2M5 11V9a2 2 0 012-2m0 0V5a2 2 0 012-2h6a2 2 0 012 2v2M7 7h10" /></svg>', color: 'purple', - keywords: ['snowflake', 'databricks', 'teradata', 'vertica', 'trino', 'presto', 'dbt'], + keywords: ['snowflake', 'databricks', 'teradata', 'vertica', 'trino', 'presto', 'dbt', 'exasol', 'clickhouse'], description: 'Snowflake, Databricks, and analytics platforms', }, { id: 'messaging', name: 'Messaging & Notifications', icon: '<svg fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 12h.01M12 12h.01M16 12h.01M21 12c0 4.418-4.03 8-9 8a9.863 9.863 0 01-4.255-.949L3 20l1.395-3.72C3.512 15.042 3 13.574 3 12c0-4.418 4.03-8 9-8s9 3.582 9 8z" /></svg>', color: 'green', - keywords: ['slack', 'sendgrid', 'smtp', 'discord', 'telegram', 'pagerduty', 'opsgenie', 'twilio'], + keywords: ['slack', 'sendgrid', 'smtp', 'discord', 'telegram', 'pagerduty', 'opsgenie', 'twilio', 'dingtalk', 'apprise'], description: 'Slack, email, SMS, and alerting services', }, { id: 'ai-ml', name: 'AI & Machine Learning', icon: '<svg fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9.663 17h4.673M12 3v1m6.364 1.636l-.707.707M21 12h-1M4 12H3m3.343-5.657l-.707-.707m2.828 9.9a5 5 0 117.072 0l-.548.547A3.374 3.374 0 0014 18.469V19a2 2 0 11-4 0v-.531c0-.895-.356-1.754-.988-2.386l-.548-.547z" /></svg>', color: 'amber', - keywords: ['openai', 'cohere', 'anthropic', 'huggingface', 'mlflow', 'pinecone', 'qdrant', 'weaviate', 'pgvector'], + keywords: ['openai', 'cohere', 'anthropic', 'huggingface', 'mlflow', 'pinecone', 'qdrant', 'weaviate', 'pgvector', 'langchain', 'llamaindex', 'mcp', 'pydantic ai', 'pydantic-ai'], Review Comment: `'pydantic-ai'` matches nothing. `fuzzyIncludes` compares raw strings, so neither direction connects the hyphen to the space in the integration name "Pydantic AI", and `'pydantic ai'` alone already covers it. Normalizing `[-_\s]` inside `fuzzyIncludes` would let one keyword handle both spellings and save the next person the same double entry. ########## registry/src/_data/providerKeywordMatch.js: ########## @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Shared by providerCategoryMap.js and exploreCategoryProviders.js so the +// two directions of the same provider <-> explore-category matching (used +// respectively by the per-provider page's category chips and the Explore +// landing page's per-category provider listing) can't drift apart. + +// Two arbitrary strings "match" a keyword if either contains the other +// (case-insensitively), mirroring the original id-only substring check. +function fuzzyIncludes(value, keyword) { + if (!value) { + return false; + } + const normalizedValue = value.toLowerCase(); + const normalizedKeyword = keyword.toLowerCase(); + return normalizedValue.includes(normalizedKeyword) || normalizedKeyword.includes(normalizedValue); Review Comment: The `normalizedKeyword.includes(normalizedValue)` half was safe when the left side was always a slug, but it now runs against free-text integration names, and the tree has 3-4 char ones ("Tpt", "Ttu", "YDB", "Bteq") that a longer keyword could swallow. I checked forward-only matching across all 105 `provider.yaml` files and category membership comes out identical, so nothing depends on the reverse direction today. Worth dropping it, or keeping it scoped to `provider.id`. ########## registry/src/_data/providerKeywordMatch.js: ########## @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Shared by providerCategoryMap.js and exploreCategoryProviders.js so the +// two directions of the same provider <-> explore-category matching (used +// respectively by the per-provider page's category chips and the Explore +// landing page's per-category provider listing) can't drift apart. + +// Two arbitrary strings "match" a keyword if either contains the other +// (case-insensitively), mirroring the original id-only substring check. +function fuzzyIncludes(value, keyword) { + if (!value) { + return false; + } + const normalizedValue = value.toLowerCase(); + const normalizedKeyword = keyword.toLowerCase(); + return normalizedValue.includes(normalizedKeyword) || normalizedKeyword.includes(normalizedValue); +} + +// Every string a provider is searchable by: its id/slug, its declared +// integration names (provider.categories[].name — e.g. "LangChain", +// "Pydantic AI"), and, once populated, each connection type's declared +// external integrations. The latter field doesn't exist in the generated +// data yet, so this reads as an empty list until a future change in +// dev/registry/extract_metadata.py starts populating it — no further +// changes needed here when that lands. +function collectSearchableValues(provider) { + const values = [provider.id]; + for (const category of provider.categories || []) { + if (category.name) { + values.push(category.name); Review Comment: I ran the new matcher over all 105 in-tree `provider.yaml` files. Beyond the intended additions it also pulls the umbrella providers into categories they weren't in before: `amazon` goes 1 category to 4 (Databases via "Amazon OpenSearch Serverless", Data Processing via "Amazon Managed Service for Apache Flink", Workflow via "Amazon Elastic Kubernetes Service (EKS)"), `google` 1 to 3, `alibaba` and `cncf-kubernetes` 1 to 2. The counts on the Explore cards move too: Databases 12 to 17, Data Processing 8 to 12, Workflow 8 to 11. Is that intended? The commit message lists the small providers but not these, and amazon has ~50 integrations, so most future keyword additions will pull it in as well. ########## registry/src/_data/providerCategoryMap.js: ########## @@ -19,14 +19,15 @@ const providersData = require("./providers.json"); const exploreCategories = require("./exploreCategories"); +const { providerMatchesKeyword } = require("./providerKeywordMatch"); module.exports = function () { const map = {}; for (const provider of providersData.providers) { const cats = []; for (const category of exploreCategories) { for (const keyword of category.keywords) { - if (provider.id.includes(keyword) || keyword.includes(provider.id)) { + if (providerMatchesKeyword(provider, keyword)) { Review Comment: This fixes the category dropdown on /providers/ (through `data-categories`), but the search box on that page still matches `data-name` only (`matchesSearch` in `src/js/provider-filters.js`), so typing "langchain" there is still a miss. If the search half of the description is in scope, `providers.njk` needs the integration names in a data attribute too. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
