This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 2f5a3762d1c Evaluate AGENTS.md and skills with Codex (#70120)
2f5a3762d1c is described below
commit 2f5a3762d1c2ba5dcb151b54d6566f259254e021
Author: LI,JHE-CHEN <[email protected]>
AuthorDate: Wed Aug 12 16:45:59 2026 +0800
Evaluate AGENTS.md and skills with Codex (#70120)
* Evaluate AGENTS.md and skills with Codex
* Skip Codex skill eval in bulk manual prek runs
---
.pre-commit-config.yaml | 21 ++++
AGENTS.md | 8 +-
contributing-docs/08_static_code_checks.rst | 2 +-
dev/skill-evals/README.md | 49 ++++++++-
dev/skill-evals/eval.py | 120 +++++++++++++++++----
dev/skill-evals/last-eval-hash.txt | 2 +-
scripts/tests/ci/prek/test_skill_eval.py | 158 ++++++++++++++++++++++++++++
7 files changed, 326 insertions(+), 34 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4c24bcc2b1d..7f8a908bb8e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1035,6 +1035,27 @@ repos:
pass_filenames: false
require_serial: true
verbose: true
+ - id: run-skill-eval-codex
+ name: Run skill-eval against AGENTS.md and cases (Codex runtime)
+ # Kept separate from run-skill-eval so only contributors who actually
run
+ # the Codex arm build an env carrying the Codex CLI binary. promptfoo
+ # bundles its own @openai/codex-sdk, but that build ships a binary
whose
+ # Developer ID certificate Apple has revoked — macOS kills it on exec —
+ # so a notarized version has to be installed explicitly.
+ entry: env AGENT_RUNTIME=codex ./dev/skill-evals/eval.py
+ language: node
+ language_version: '22.22.0'
+ additional_dependencies:
+ - '[email protected]'
+ - '@openai/[email protected]'
+ stages: ['manual']
+ files: >
+ (?x)
+ ^AGENTS\.md$|
+ ^dev/skill-evals/
+ pass_filenames: false
+ require_serial: true
+ verbose: true
- id: view-skill-eval
name: View skill-eval results in browser (manual)
entry: env PROMPTFOO_CONFIG_DIR=.build/promptfoo promptfoo view
diff --git a/AGENTS.md b/AGENTS.md
index 2086b4040ad..c414057ac1f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -52,7 +52,7 @@ Don't spell out **Directed Acyclic Graph** except for
historical context.
- **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`
- **Format with ruff only:** `prek run ruff-format --from-ref <target_branch>`
- **Run regular (fast) static checks:** `prek run --from-ref <target_branch>
--stage pre-commit`
-- **Run manual (slower) checks:** `prek run --from-ref <target_branch> --stage
manual --skip compile-ui-assets-dev --skip view-skill-eval` (the skipped hooks
start long-running local servers rather than checks that complete)
+- **Run manual (slower) checks:** `prek run --from-ref <target_branch> --stage
manual --skip compile-ui-assets-dev --skip view-skill-eval --skip
run-skill-eval-codex` (the skipped hooks start long-running local servers or
provision the opt-in Codex environment rather than run checks that complete)
- **Build docs:** `breeze build-docs`
- **Determine which tests to run based on changed files:** `breeze ci
selective-check --commit-ref <commit_with_squashed_changes>`
<!-- END generated-commands, please keep comment here to allow auto update -->
@@ -295,9 +295,9 @@ code review checklist in
[`.github/instructions/code-review.instructions.md`](.g
4. Run regular (fast) static checks (`prek run --from-ref <target_branch>
--stage pre-commit`)
and fix any failures. This includes mypy checks for non-provider projects
(airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common).
5. Run manual (slower) checks
- (`prek run --from-ref <target_branch> --stage manual --skip
compile-ui-assets-dev --skip view-skill-eval`)
- and fix any failures. The skipped hooks start long-running local servers
rather than checks
- that complete.
+ (`prek run --from-ref <target_branch> --stage manual --skip
compile-ui-assets-dev --skip view-skill-eval --skip run-skill-eval-codex`)
+ and fix any failures. The skipped hooks start long-running local servers or
provision the
+ opt-in Codex environment rather than run checks that complete.
6. Run relevant individual tests and confirm they pass.
7. Find which tests to run for the changes with selective-checks and run those
tests in parallel to confirm they pass and check for CI-specific issues.
8. Check for security issues — no secrets, no injection vulnerabilities, no
unsafe patterns.
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index 161bd7902b8..9bae6b9248f 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -234,7 +234,7 @@ To check other usage types of the pre-commit framework, see
`Pre-commit website
- "- **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`"
- "- **Format with ruff only:** `prek run ruff-format --from-ref
<target_branch>`"
- "- **Run regular (fast) static checks:** `prek run --from-ref
<target_branch> --stage pre-commit`"
- - "- **Run manual (slower) checks:** `prek run --from-ref <target_branch>
--stage manual --skip compile-ui-assets-dev --skip view-skill-eval` (the
skipped hooks start long-running local servers rather than checks that
complete)"
+ - "- **Run manual (slower) checks:** `prek run --from-ref <target_branch>
--stage manual --skip compile-ui-assets-dev --skip view-skill-eval --skip
run-skill-eval-codex` (the skipped hooks start long-running local servers or
provision the opt-in Codex environment rather than run checks that complete)"
- "- **Build docs:** `breeze build-docs`"
- "- **Determine which tests to run based on changed files:** `breeze ci
selective-check --commit-ref <commit_with_squashed_changes>`"
.. AGENT-SKILL-END
diff --git a/dev/skill-evals/README.md b/dev/skill-evals/README.md
index 57748b96c2e..e25333ff89a 100644
--- a/dev/skill-evals/README.md
+++ b/dev/skill-evals/README.md
@@ -23,6 +23,7 @@
- [Skill-Eval Harness](#skill-eval-harness)
- [Prerequisites](#prerequisites)
+ - [Agent runtimes](#agent-runtimes)
- [Usage](#usage)
- [Cleanup](#cleanup)
- [Adding cases](#adding-cases)
@@ -48,12 +49,49 @@ regular-file CLAUDE.md would make every arm read identical
guidance.
## Prerequisites
-- **Authentication** (one of):
+- **Claude authentication** (one of):
- Claude Code session (`claude /login`) — Pro/Max subscription
- `ANTHROPIC_API_KEY` environment variable — API credits
+- **Codex authentication** (when using the Codex runtime): a Codex CLI
+ session (`codex login`)
-That's it — prek provisions Node, promptfoo, and the Claude Agent SDK
-automatically.
+That's it — prek provisions Node, promptfoo, and the agent SDKs automatically.
+
+## Agent runtimes
+
+Claude is the default runtime. The `run-skill-eval-codex` hook runs the same
+arms and cases through the official Codex SDK instead. The Codex provider uses
+a fresh thread for every prompt, read-only sandboxing, disabled network access,
+disabled session-history persistence, and the SDK's structured-output support.
+
+The two runtimes are separate hooks, not one hook with a flag, because the
+Codex env carries a ~300 MB Codex CLI binary. Keeping it separate means prek
+only builds that env for contributors who actually run the Codex arm. promptfoo
+bundles its own older `@openai/codex-sdk`, but that build ships a binary whose
+Developer ID certificate Apple has revoked — macOS kills it on exec and reports
+it as malware — so the Codex hook installs a notarized version explicitly.
+
+Codex caps the project docs it reads at `project_doc_max_bytes` and truncates
+past it, and the 32,768-byte default is smaller than `AGENTS.md`. The harness
+raises that cap so the whole file reaches the model — otherwise guidance
+appended at the end would be invisible to every arm and the eval would compare
+identical prefixes.
+
+When `SKILL_NAME` is set, both runtimes verify skill usage with promptfoo's
+`skill-used` assertion. The Codex SDK does not expose a first-class skill-use
+event, so promptfoo infers usage from successful reads of the skill's
+`SKILL.md` file.
+
+```bash
+# Default: Claude Agent SDK
+prek run run-skill-eval --hook-stage manual --all-files
+
+# Codex SDK, using its configured default model
+prek run run-skill-eval-codex --hook-stage manual --all-files
+
+# Codex SDK with an explicit model
+MODEL=gpt-5.4 prek run run-skill-eval-codex --hook-stage manual --all-files
+```
## Usage
@@ -125,8 +163,9 @@ Use `output.should_create` directly in assertions.
1. Creates git worktrees — one with `main`'s AGENTS.md, one with
your working tree version. Both are full repo checkouts.
2. Generates a [promptfoo](https://github.com/promptfoo/promptfoo)
- config with `anthropic:claude-agent-sdk` provider and
- `output_format: json_schema` for structured output.
+ config for the selected runtime. Claude uses promptfoo's
+ `anthropic:claude-agent-sdk` provider; Codex uses its `openai:codex-sdk`
+ provider. Both request schema-constrained structured output.
3. Runs each case against all arms in parallel.
4. Reports pass/fail diff. Worktrees cleaned up on exit.
diff --git a/dev/skill-evals/eval.py b/dev/skill-evals/eval.py
index 20eb766862a..19184629994 100755
--- a/dev/skill-evals/eval.py
+++ b/dev/skill-evals/eval.py
@@ -26,12 +26,15 @@ real repo — the agent sees actual source files.
Usage:
prek run run-skill-eval --hook-stage manual --all-files
+ prek run run-skill-eval-codex --hook-stage manual --all-files
-Env knobs: MODEL, SKILL_NAME, EVAL_REPEAT, EVAL_FULL (baseline arm).
+Env knobs: AGENT_RUNTIME (set by the Codex hook), MODEL, SKILL_NAME,
EVAL_REPEAT,
+EVAL_FULL (baseline arm).
Promptfoo flags like --filter* are argv-only — wire them as fixed entry
args on a hook variant when needed.
-Authentication: Claude Code session (claude /login) or ANTHROPIC_API_KEY.
+Authentication: Claude Code session (claude /login), ANTHROPIC_API_KEY,
+or a Codex CLI session (codex login), depending on the selected runtime.
"""
from __future__ import annotations
@@ -45,6 +48,16 @@ import tempfile
from pathlib import Path
PROMPTFOO_VERSION = "0.121.17"
+SDK_PACKAGES = {
+ "claude": "@anthropic-ai/claude-agent-sdk",
+ "codex": "@openai/codex-sdk",
+}
+SUPPORTED_RUNTIMES = tuple(SDK_PACKAGES)
+
+# Codex prefix-truncates project docs past project_doc_max_bytes, and the
32,768-byte
+# default is already smaller than AGENTS.md — guidance appended at the end
would be
+# invisible to every arm while the eval still recorded a hash as proof it ran.
+CODEX_PROJECT_DOC_MAX_BYTES = 1_048_576
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
SCRIPT_DIR = Path(__file__).resolve().parent
@@ -65,7 +78,7 @@ OUTPUT_FORMAT = {
"type": "json_schema",
"schema": {
"type": "object",
- "required": ["should_create", "rationale"],
+ "required": ["should_create", "type", "rationale"],
"additionalProperties": False,
"properties": {
"should_create": {"type": "boolean"},
@@ -83,12 +96,20 @@ def run(cmd: list[str], **kwargs) ->
subprocess.CompletedProcess[str]:
return subprocess.run(cmd, capture_output=True, text=True, check=False,
**kwargs)
-def find_sdk_modules() -> Path:
- """Locate the node_modules dir (in the prek env) that contains the Claude
Agent SDK.
+def find_sdk_modules(runtime: str) -> Path:
+ """Locate the prek node_modules dir containing the selected agent SDK.
- promptfoo resolves the SDK from the eval config's directory, not from its
- own install tree — the caller symlinks this dir next to the config.
+ promptfoo resolves providers from the eval config's directory. The caller
+ symlinks this directory next to the generated config.
+
+ Only the hook's own install root counts, never promptfoo's nested
+ node_modules: promptfoo declares both SDKs as optional dependencies, so a
+ prek env holds two copies of each, and its bundled Codex build ships a
+ binary whose Developer ID certificate Apple has revoked — macOS kills it on
+ exec. Resolving by location rather than by version keeps the pins in
+ .pre-commit-config.yaml the single source of truth.
"""
+ sdk_package = SDK_PACKAGES[runtime]
promptfoo_bin = shutil.which("promptfoo")
if promptfoo_bin:
# PROMPTFOO_DISABLE_UPDATE avoids an outdated version banner on stdout
when a newer
@@ -102,12 +123,13 @@ def find_sdk_modules() -> Path:
if pf_pkg.parent == pf_pkg:
break
pf_pkg = pf_pkg.parent
- for candidate in (pf_pkg / "node_modules", pf_pkg.parent):
- if (candidate / "@anthropic-ai" / "claude-agent-sdk").is_dir():
- return candidate
+ install_root = pf_pkg.parent
+ if install_root.joinpath(*sdk_package.split("/"),
"package.json").is_file():
+ return install_root
+ hook = "run-skill-eval-codex" if runtime == "codex" else "run-skill-eval"
print(
- "Error: promptfoo with the Claude Agent SDK not found. Run the eval
via:\n"
- " prek run run-skill-eval --hook-stage manual --all-files",
+ f"Error: promptfoo {PROMPTFOO_VERSION} with {sdk_package} not found.
Run the eval via:\n"
+ f" prek run {hook} --hook-stage manual --all-files",
file=sys.stderr,
)
sys.exit(1)
@@ -207,20 +229,56 @@ def remove_worktree(wt_dir: Path) -> None:
run(["git", "-C", str(REPO_ROOT), "worktree", "remove", "--force",
str(wt_dir)])
-def build_provider(label: str, working_dir: Path, model: str, skill_name: str
| None = None) -> dict:
+def build_provider(label: str, working_dir: Path, model: str | None,
skill_name: str | None = None) -> dict:
config: dict = {
- "model": model,
"apiKeyRequired": False,
"setting_sources": ["project"],
"append_allowed_tools": ["Read", "Grep", "Glob"],
"working_dir": str(working_dir),
"output_format": OUTPUT_FORMAT,
}
+ if model:
+ config["model"] = model
if skill_name:
config["skills"] = [skill_name]
return {"id": "anthropic:claude-agent-sdk", "label": label, "config":
config}
+def build_codex_provider(
+ label: str, working_dir: Path, model: str | None, detect_skill_usage: bool
= False
+) -> dict:
+ config = {
+ "approval_policy": "never",
+ "cli_config": {
+ "history": {"persistence": "none"},
+ "project_doc_max_bytes": CODEX_PROJECT_DOC_MAX_BYTES,
+ },
+ "network_access_enabled": False,
+ "output_schema": OUTPUT_FORMAT["schema"],
+ "sandbox_mode": "read-only",
+ "web_search_mode": "disabled",
+ "working_dir": str(working_dir),
+ }
+ if model:
+ config["model"] = model
+ if detect_skill_usage:
+ config["enable_streaming"] = True
+ return {
+ "id": "openai:codex-sdk",
+ "label": label,
+ "config": config,
+ "transform": "JSON.parse(output)",
+ }
+
+
+def get_runtime() -> str:
+ runtime = os.environ.get("AGENT_RUNTIME", "claude").lower()
+ if runtime not in SUPPORTED_RUNTIMES:
+ choices = ", ".join(SUPPORTED_RUNTIMES)
+ raise ValueError(f"AGENT_RUNTIME must be one of: {choices}; got
{runtime!r}")
+ return runtime
+
+
def count_provider_errors(results_file: Path) -> int:
"""Count results whose provider call errored (as opposed to failing an
assertion)."""
try:
@@ -231,9 +289,16 @@ def count_provider_errors(results_file: Path) -> int:
def main() -> int:
- sdk_modules = find_sdk_modules()
+ try:
+ runtime = get_runtime()
+ except ValueError as error:
+ print(f"Error: {error}", file=sys.stderr)
+ return 1
+ sdk_modules = find_sdk_modules(runtime)
- model = os.environ.get("MODEL", "claude-sonnet-4-6")
+ model = os.environ.get("MODEL")
+ if not model and runtime == "claude":
+ model = "claude-sonnet-4-6"
skill_name = os.environ.get("SKILL_NAME")
skill_src = None
if skill_name:
@@ -263,7 +328,8 @@ def main() -> int:
return 1
base_branch = resolve_base_branch()
- check_claude_md_symlink(base_branch)
+ if runtime == "claude":
+ check_claude_md_symlink(base_branch)
# Hash before building arms so edits made mid-run aren't recorded as
tested.
guidance_hash = compute_guidance_hash(AGENTS_SRC, CASES_DIR)
@@ -273,7 +339,7 @@ def main() -> int:
worktrees: list[Path] = []
try:
- # promptfoo resolves the Claude Agent SDK from the config directory
+ # promptfoo resolves the selected agent SDK from the config directory
(work_dir / "node_modules").symlink_to(sdk_modules)
# Extract main-branch AGENTS.md
@@ -319,11 +385,16 @@ def main() -> int:
arm_baseline = create_worktree(work_dir, "baseline", base_branch,
None, worktrees)
# Generate config (JSON — valid promptfoo config, keeps the script
stdlib-only)
- providers = [build_provider("main", arm_main, model, skill_name)]
+ def provider(label: str, arm: Path, selected_skill: str | None = None)
-> dict:
+ if runtime == "codex":
+ return build_codex_provider(label, arm, model,
detect_skill_usage=bool(selected_skill))
+ return build_provider(label, arm, model, selected_skill)
+
+ providers = [provider("main", arm_main, skill_name)]
if arm_working:
- providers.append(build_provider("working", arm_working, model,
skill_name))
+ providers.append(provider("working", arm_working, skill_name))
if full_mode and arm_baseline:
- providers.append(build_provider("baseline", arm_baseline, model))
+ providers.append(provider("baseline", arm_baseline))
default_test: dict = {"options": {"disableVarExpansion": True}}
if skill_name:
@@ -339,10 +410,13 @@ def main() -> int:
config_path.write_text(json.dumps(config, indent=2))
# Report
+ model_label = model or "runtime default"
if skill_name:
- print(f"Mode: {len(providers)} arms, skill '{skill_name}', model:
{model}")
+ print(
+ f"Mode: {len(providers)} arms, skill '{skill_name}', runtime:
{runtime}, model: {model_label}"
+ )
else:
- print(f"Mode: {len(providers)} arms, AGENTS.md only, model:
{model}")
+ print(f"Mode: {len(providers)} arms, AGENTS.md only, runtime:
{runtime}, model: {model_label}")
print()
print(f"Changes detected (vs {base_branch}):")
diff --git a/dev/skill-evals/last-eval-hash.txt
b/dev/skill-evals/last-eval-hash.txt
index 23452e4e6f3..4e72dc42b31 100644
--- a/dev/skill-evals/last-eval-hash.txt
+++ b/dev/skill-evals/last-eval-hash.txt
@@ -1,3 +1,3 @@
# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by
hand.
# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate.
-725f2a0640c5b6fec7f6666484a0577dde5c87f19206b1d0bdf191a7094cd98d
+8cfe725b810c4c2d364d99f63b27ec739545e40032195624c218b0eb23d75859
diff --git a/scripts/tests/ci/prek/test_skill_eval.py
b/scripts/tests/ci/prek/test_skill_eval.py
new file mode 100644
index 00000000000..8bac034debb
--- /dev/null
+++ b/scripts/tests/ci/prek/test_skill_eval.py
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+MODULE_PATH = REPO_ROOT / "dev" / "skill-evals" / "eval.py"
+
+
[email protected](scope="module")
+def skill_eval_module():
+ spec = importlib.util.spec_from_file_location("skill_eval", MODULE_PATH)
+ assert spec and spec.loader
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected](
+ ("configured", "expected"),
+ [
+ pytest.param(None, "claude", id="default"),
+ pytest.param("Claude", "claude", id="case-insensitive"),
+ pytest.param("codex", "codex", id="codex"),
+ ],
+)
+def test_get_runtime(monkeypatch, skill_eval_module, configured, expected):
+ if configured is None:
+ monkeypatch.delenv("AGENT_RUNTIME", raising=False)
+ else:
+ monkeypatch.setenv("AGENT_RUNTIME", configured)
+
+ assert skill_eval_module.get_runtime() == expected
+
+
+def test_get_runtime_rejects_unknown_runtime(monkeypatch, skill_eval_module):
+ monkeypatch.setenv("AGENT_RUNTIME", "unknown")
+
+ with pytest.raises(ValueError, match="claude, codex"):
+ skill_eval_module.get_runtime()
+
+
+def test_build_codex_provider(skill_eval_module, tmp_path):
+ working_dir = tmp_path / "worktree"
+
+ provider = skill_eval_module.build_codex_provider(
+ "working", working_dir, "gpt-test", detect_skill_usage=True
+ )
+
+ assert provider == {
+ "id": "openai:codex-sdk",
+ "label": "working",
+ "config": {
+ "approval_policy": "never",
+ "cli_config": {
+ "history": {"persistence": "none"},
+ "project_doc_max_bytes":
skill_eval_module.CODEX_PROJECT_DOC_MAX_BYTES,
+ },
+ "enable_streaming": True,
+ "model": "gpt-test",
+ "network_access_enabled": False,
+ "output_schema": skill_eval_module.OUTPUT_FORMAT["schema"],
+ "sandbox_mode": "read-only",
+ "web_search_mode": "disabled",
+ "working_dir": str(working_dir),
+ },
+ "transform": "JSON.parse(output)",
+ }
+
+
+def test_output_schema_requires_every_property(skill_eval_module):
+ schema = skill_eval_module.OUTPUT_FORMAT["schema"]
+
+ assert set(schema["required"]) == set(schema["properties"])
+
+
+def test_build_codex_provider_uses_runtime_default_model(skill_eval_module,
tmp_path):
+ provider = skill_eval_module.build_codex_provider("main", tmp_path /
"worktree", None)
+
+ assert "model" not in provider["config"]
+ assert "enable_streaming" not in provider["config"]
+
+
[email protected]
+def promptfoo_layout(monkeypatch, skill_eval_module, tmp_path):
+ """Build a prek-style node env and return an SDK installer for either
location.
+
+ ``install_root`` mirrors a package listed in the hook's
+ additional_dependencies; ``bundled`` mirrors promptfoo's own optional
+ dependency, nested inside its package.
+ """
+ install_root = tmp_path / "node_modules"
+ promptfoo = install_root / "promptfoo"
+ bundled = promptfoo / "node_modules"
+ promptfoo.mkdir(parents=True)
+ (promptfoo / "package.json").write_text("{}")
+ promptfoo_bin = promptfoo / "promptfoo"
+ promptfoo_bin.touch()
+ monkeypatch.setattr(skill_eval_module.shutil, "which", lambda _:
str(promptfoo_bin))
+ monkeypatch.setattr(
+ skill_eval_module,
+ "run",
+ lambda *args, **kwargs:
SimpleNamespace(stdout=f"{skill_eval_module.PROMPTFOO_VERSION}\n"),
+ )
+
+ def install(location: Path, package: str) -> None:
+ sdk = location.joinpath(*package.split("/"))
+ sdk.mkdir(parents=True)
+ (sdk / "package.json").write_text('{"version": "1.2.3"}')
+
+ return SimpleNamespace(install_root=install_root, bundled=bundled,
install=install)
+
+
[email protected]("runtime", ["claude", "codex"])
+def test_find_sdk_modules_resolves_the_hook_installed_copy(skill_eval_module,
promptfoo_layout, runtime):
+ promptfoo_layout.install(promptfoo_layout.install_root,
skill_eval_module.SDK_PACKAGES[runtime])
+
+ assert skill_eval_module.find_sdk_modules(runtime) ==
promptfoo_layout.install_root
+
+
[email protected]("runtime", ["claude", "codex"])
+def test_find_sdk_modules_ignores_the_copy_bundled_with_promptfoo(
+ skill_eval_module, promptfoo_layout, runtime
+):
+ promptfoo_layout.install(promptfoo_layout.bundled,
skill_eval_module.SDK_PACKAGES[runtime])
+
+ with pytest.raises(SystemExit) as exit_info:
+ skill_eval_module.find_sdk_modules(runtime)
+
+ assert exit_info.value.code == 1
+
+
+def
test_find_sdk_modules_requires_the_selected_runtimes_sdk(skill_eval_module,
promptfoo_layout):
+ promptfoo_layout.install(promptfoo_layout.install_root,
skill_eval_module.SDK_PACKAGES["claude"])
+
+ with pytest.raises(SystemExit) as exit_info:
+ skill_eval_module.find_sdk_modules("codex")
+
+ assert exit_info.value.code == 1
