Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package python-langsmith for
openSUSE:Factory checked in at 2026-07-24 22:05:15
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-langsmith (Old)
and /work/SRC/openSUSE:Factory/.python-langsmith.new.2004 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-langsmith"
Fri Jul 24 22:05:15 2026 rev:14 rq:1367619 version:0.10.10
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-langsmith/python-langsmith.changes
2026-07-22 19:08:42.980479020 +0200
+++
/work/SRC/openSUSE:Factory/.python-langsmith.new.2004/python-langsmith.changes
2026-07-24 22:05:21.649747222 +0200
@@ -1,0 +2,9 @@
+Thu Jul 23 19:49:50 UTC 2026 - Martin Pluskal <[email protected]>
+
+- Update to 0.10.10:
+ * Require a project to be specified when calling get_run_stats()
+ * Add session_id and start_time to feedback generated from
+ evaluations
+ * Merge sandbox request headers case-insensitively
+
+-------------------------------------------------------------------
Old:
----
langsmith-0.10.9.tar.gz
New:
----
langsmith-0.10.10.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-langsmith.spec ++++++
--- /var/tmp/diff_new_pack.pfJRDm/_old 2026-07-24 22:05:23.021795136 +0200
+++ /var/tmp/diff_new_pack.pfJRDm/_new 2026-07-24 22:05:23.025795276 +0200
@@ -17,7 +17,7 @@
Name: python-langsmith
-Version: 0.10.9
+Version: 0.10.10
Release: 0
Summary: Client library for the LangSmith LLM tracing and evaluation
platform
License: MIT
++++++ langsmith-0.10.9.tar.gz -> langsmith-0.10.10.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/.bumpversion.cfg
new/langsmith-0.10.10/.bumpversion.cfg
--- old/langsmith-0.10.9/.bumpversion.cfg 2020-02-02 01:00:00.000000000
+0100
+++ new/langsmith-0.10.10/.bumpversion.cfg 2020-02-02 01:00:00.000000000
+0100
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.10.9
+current_version = 0.10.10
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
serialize = {major}.{minor}.{patch}
search = {current_version}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/PKG-INFO
new/langsmith-0.10.10/PKG-INFO
--- old/langsmith-0.10.9/PKG-INFO 2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/PKG-INFO 2020-02-02 01:00:00.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 2.4
Name: langsmith
-Version: 0.10.9
+Version: 0.10.10
Summary: Client library to connect to the LangSmith Observability and
Evaluation Platform.
Project-URL: Homepage, https://smith.langchain.com/
Project-URL: Documentation, https://docs.smith.langchain.com/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/__init__.py
new/langsmith-0.10.10/langsmith/__init__.py
--- old/langsmith-0.10.9/langsmith/__init__.py 2020-02-02 01:00:00.000000000
+0100
+++ new/langsmith-0.10.10/langsmith/__init__.py 2020-02-02 01:00:00.000000000
+0100
@@ -49,7 +49,7 @@
# Avoid calling into importlib on every call to __version__
-__version__ = "0.10.9"
+__version__ = "0.10.10"
version = __version__ # for backwards compatibility
# Metadata key to hide a traced run from LangSmith's Messages View.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/langsmith/_internal/_v2_migration_utils.py
new/langsmith-0.10.10/langsmith/_internal/_v2_migration_utils.py
--- old/langsmith-0.10.9/langsmith/_internal/_v2_migration_utils.py
1970-01-01 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/langsmith/_internal/_v2_migration_utils.py
2020-02-02 01:00:00.000000000 +0100
@@ -0,0 +1,84 @@
+"""Utilities for migrating functionality to the v2 LangSmith API."""
+
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING, Any, Optional
+
+from langsmith import schemas
+
+if TYPE_CHECKING:
+ from langsmith.client import Client
+
+
+# Fields for `/v2/runs/query` (RunSelectField enum); omitting selects returns
only id.
+_V2_RUN_SELECTS = [
+ "ID",
+ "NAME",
+ "RUN_TYPE",
+ "STATUS",
+ "START_TIME",
+ "END_TIME",
+ "INPUTS",
+ "OUTPUTS",
+ "PARENT_RUN_IDS",
+ "PROJECT_ID",
+ "TRACE_ID",
+ "DOTTED_ORDER",
+ "REFERENCE_EXAMPLE_ID",
+ "ERROR",
+]
+
+
+def _load_traces_v2(
+ project: schemas.TracerSession,
+ client: Client,
+ *,
+ is_root: Optional[bool],
+) -> list[schemas.Run]:
+ """List an experiment's runs from v2.
+
+ `query_v2` defaults `min_start_time` to ~24h, so bound the window to the
session
+ explicitly or older experiments drop.
+ """
+ now = datetime.datetime.now(datetime.timezone.utc)
+ kwargs: dict[str, Any] = {
+ "project_ids": [str(project.id)],
+ "min_start_time": project.start_time,
+ "max_start_time": project.end_time or now,
+ "selects": _V2_RUN_SELECTS,
+ }
+ if is_root is not None:
+ kwargs["is_root"] = is_root
+ pager = client._get_langsmith_api_sync().runs.query_v2(**kwargs)
+ return [_v2_run_to_schema(run) for run in pager]
+
+
+def _v2_run_to_schema(run: Any) -> schemas.Run:
+ """Map a v2 `Run` to `schemas.Run`.
+
+ `project_id`→`session_id`, `parent_run_ids[-1]`→`parent_run_id`; drop
`None` so
+ schema defaults apply (e.g. `dotted_order`).
+ """
+ parent_run_ids = getattr(run, "parent_run_ids", None)
+ fields = {
+ "id": run.id,
+ "name": run.name,
+ "run_type": run.run_type,
+ "start_time": run.start_time,
+ "end_time": getattr(run, "end_time", None),
+ "trace_id": run.trace_id,
+ "session_id": getattr(run, "project_id", None),
+ "parent_run_id": parent_run_ids[-1] if parent_run_ids else None,
+ "dotted_order": getattr(run, "dotted_order", None),
+ "reference_example_id": getattr(run, "reference_example_id", None),
+ "inputs": getattr(run, "inputs", None) or {},
+ "outputs": getattr(run, "outputs", None),
+ "error": getattr(run, "error", None),
+ "status": (
+ run.status.lower() if getattr(run, "status", None) is not None
else None
+ ),
+ }
+ return schemas.Run(
+ **{key: value for key, value in fields.items() if value is not None}
+ )
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/client.py
new/langsmith-0.10.10/langsmith/client.py
--- old/langsmith-0.10.9/langsmith/client.py 2020-02-02 01:00:00.000000000
+0100
+++ new/langsmith-0.10.10/langsmith/client.py 2020-02-02 01:00:00.000000000
+0100
@@ -113,6 +113,7 @@
from langsmith._internal._serde import dumps_json as _dumps_json
from langsmith._internal._uuid import uuid7
from langsmith._openapi_client import AsyncLangsmith as LangsmithOpenAPIClient
+from langsmith._openapi_client import Langsmith as SyncLangsmithOpenAPIClient
from langsmith.prompt_cache import PromptCache, prompt_cache_singleton
from langsmith.schemas import AttachmentInfo, ExampleWithRuns
@@ -917,6 +918,7 @@
"_profile_auth",
"_profile_auth_headers",
"_langsmith_api",
+ "_langsmith_api_sync",
]
_api_key: Optional[str]
@@ -928,6 +930,7 @@
_profile_auth: Optional[_profiles.ProfileAuth]
_profile_auth_headers: dict[str, str]
_langsmith_api: Optional[LangsmithOpenAPIClient]
+ _langsmith_api_sync: Optional[SyncLangsmithOpenAPIClient]
def __init__(
self,
@@ -1467,6 +1470,7 @@
self._failed_traces_max_bytes = 100 * 1024 * 1024
self._langsmith_api = None
+ self._langsmith_api_sync = None
# ------------------------------------------------------------------
# Stainless v2 resource accessors
@@ -1491,6 +1495,22 @@
)
return self._langsmith_api
+ def _get_langsmith_api_sync(self) -> SyncLangsmithOpenAPIClient:
+ if self._langsmith_api_sync is None:
+ self._langsmith_api_sync = SyncLangsmithOpenAPIClient(
+ api_key=self._api_key,
+ tenant_id=str(self._workspace_id) if self._workspace_id else
None,
+ base_url=self.api_url,
+ timeout=_httpx.Timeout(
+ connect=self._timeout[0],
+ read=self._timeout[1],
+ write=self._timeout[1],
+ pool=self._timeout[0],
+ ),
+ default_headers=self._headers or None,
+ )
+ return self._langsmith_api_sync
+
@property
def runs(self) -> AsyncRunsResource:
"""Access the runs resource."""
@@ -4468,6 +4488,10 @@
]
for future in as_completed(futures):
project_ids.append(future.result().id)
+ if not project_ids:
+ raise ValueError(
+ "At least one of project_names or project_ids must be
provided."
+ )
payload = {
"id": id,
"trace": trace,
@@ -7604,10 +7628,10 @@
Optional[ls_schemas.FeedbackConfig], res.feedback_config
),
feedback_source_type=ls_schemas.FeedbackSourceType.MODEL,
- project_id=project_id,
+ project_id=project_id if run is None else None,
extra=res.extra,
trace_id=run.trace_id if run else None,
- session_id=run.session_id if run else None,
+ session_id=run.session_id if run and run.session_id else
project_id,
start_time=run.start_time if run else None,
error=error,
)
@@ -7731,8 +7755,7 @@
The number of times to retry the request before giving up.
project_id (Optional[Union[UUID, str]]):
The ID of the project (or experiment) to provide feedback on.
This is
- used for creating summary metrics for experiments. Cannot
specify
- run_id or trace_id if project_id is specified, and vice versa.
+ used for creating summary metrics for experiments.
comparative_experiment_id (Optional[Union[UUID, str]]):
If this feedback was logged as a part of a comparative
experiment, this
associates the feedback with that experiment.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/evaluation/_arunner.py
new/langsmith-0.10.10/langsmith/evaluation/_arunner.py
--- old/langsmith-0.10.9/langsmith/evaluation/_arunner.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/langsmith/evaluation/_arunner.py 2020-02-02
01:00:00.000000000 +0100
@@ -42,7 +42,7 @@
_load_examples_map,
_load_experiment,
_load_tqdm,
- _load_traces,
+ _load_traces_for_experiment,
_resolve_data,
_resolve_evaluators,
_resolve_experiment,
@@ -447,8 +447,8 @@
)
runs = await aitertools.aio_to_thread(
contextvars.copy_context(),
- _load_traces,
- experiment,
+ _load_traces_for_experiment,
+ project,
client,
load_nested=load_nested,
)
@@ -1021,7 +1021,10 @@
if self._upload_results:
self.client._log_evaluation_feedback(
- evaluator_response, run=run,
_executor=feedback_executor
+ evaluator_response,
+ run=run,
+ project_id=self._get_experiment().id,
+ _executor=feedback_executor,
)
return selected_results
except Exception as e:
@@ -1044,7 +1047,10 @@
)
if self._upload_results:
self.client._log_evaluation_feedback(
- error_response, run=run,
_executor=feedback_executor
+ error_response,
+ run=run,
+ project_id=self._get_experiment().id,
+ _executor=feedback_executor,
)
return selected_results
except Exception as e2:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/evaluation/_runner.py
new/langsmith-0.10.10/langsmith/evaluation/_runner.py
--- old/langsmith-0.10.9/langsmith/evaluation/_runner.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/langsmith/evaluation/_runner.py 2020-02-02
01:00:00.000000000 +0100
@@ -43,6 +43,7 @@
from langsmith import run_trees as rt
from langsmith import schemas
from langsmith import utils as ls_utils
+from langsmith._internal import _v2_migration_utils
from langsmith._internal._beta_decorator import _warn_once
from langsmith.evaluation.evaluator import (
SUMMARY_EVALUATOR_T,
@@ -505,7 +506,7 @@
""" # noqa: E501
client = client or rt.get_cached_client(timeout_ms=(20_000, 90_001))
project = _load_experiment(experiment, client)
- runs = _load_traces(experiment, client, load_nested=load_nested)
+ runs = _load_traces_for_experiment(project, client,
load_nested=load_nested)
data_map = _load_examples_map(client, project)
data = [data_map[cast(uuid.UUID, run.reference_example_id)] for run in
runs]
return _evaluate(
@@ -893,8 +894,8 @@
comparison_url = _build_comparative_url(experiments_tuple,
comparative_experiment)
_print_comparative_experiment_start(comparison_url)
runs = [
- _load_traces(experiment, client, load_nested=load_nested)
- for experiment in experiments
+ _load_traces_for_experiment(project, client, load_nested=load_nested)
+ for project in projects
]
# Only check intersections for the experiments
examples_intersection = None
@@ -1160,14 +1161,20 @@
return client.read_project(project_name=project)
-def _load_traces(
+def _load_traces_for_experiment(
project: Union[str, uuid.UUID, schemas.TracerSession],
client: langsmith.Client,
load_nested: bool = False,
) -> list[schemas.Run]:
"""Load nested traces for a given project."""
is_root = None if load_nested else True
- if isinstance(project, schemas.TracerSession):
+ runs: Iterable[schemas.Run]
+ # v1 `/runs/query` returns 501 on SmithDB-only backends; use v2 when it's
available.
+ if (client.info.instance_flags or {}).get("sdb_query_enabled"):
+ if not isinstance(project, schemas.TracerSession):
+ project = _load_experiment(project, client)
+ runs = _v2_migration_utils._load_traces_v2(project, client,
is_root=is_root)
+ elif isinstance(project, schemas.TracerSession):
runs = client.list_runs(project_id=project.id, is_root=is_root)
elif isinstance(project, uuid.UUID) or _is_uuid(project):
runs = client.list_runs(project_id=project, is_root=is_root)
@@ -1681,7 +1688,10 @@
if self._upload_results:
# TODO: This is a hack
self.client._log_evaluation_feedback(
- evaluator_response, run=run, _executor=executor
+ evaluator_response,
+ run=run,
+ project_id=self._get_experiment().id,
+ _executor=executor,
)
except Exception as e:
try:
@@ -1704,7 +1714,10 @@
if self._upload_results:
# TODO: This is a hack
self.client._log_evaluation_feedback(
- error_response, run=run, _executor=executor
+ error_response,
+ run=run,
+ project_id=self._get_experiment().id,
+ _executor=executor,
)
except Exception as e2:
logger.debug(f"Error parsing feedback keys: {e2}")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/sandbox/_helpers.py
new/langsmith-0.10.10/langsmith/sandbox/_helpers.py
--- old/langsmith-0.10.9/langsmith/sandbox/_helpers.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/langsmith/sandbox/_helpers.py 2020-02-02
01:00:00.000000000 +0100
@@ -34,10 +34,17 @@
base_headers: Optional[Mapping[str, str]] = None,
override_headers: Optional[Mapping[str, str]] = None,
) -> dict[str, str]:
- """Merge request headers, giving precedence to overrides."""
- merged: dict[str, str] = dict(base_headers or {})
- if override_headers:
- merged.update(override_headers)
+ """Merge request headers, giving precedence to overrides.
+
+ Names are normalized to lowercase so an override replaces a base header
that
+ differs only in casing. HTTP header names are case-insensitive, so keeping
+ both would be ambiguous — a server reading the first value would see the
+ base header instead of the override.
+ """
+ merged: dict[str, str] = {}
+ for headers in (base_headers, override_headers):
+ for name, value in (headers or {}).items():
+ merged[name.lower()] = value
return merged
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/langsmith/testing/_internal.py
new/langsmith-0.10.10/langsmith/testing/_internal.py
--- old/langsmith-0.10.9/langsmith/testing/_internal.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/langsmith/testing/_internal.py 2020-02-02
01:00:00.000000000 +0100
@@ -628,6 +628,7 @@
def submit_result(
self,
run_id: uuid.UUID,
+ start_time: datetime.datetime,
error: Optional[str] = None,
skipped: bool = False,
pytest_plugin: Any = None,
@@ -644,12 +645,24 @@
status = "passed"
if pytest_plugin and pytest_nodeid:
pytest_plugin.update_process_status(pytest_nodeid, {"status":
status})
- self._executor.submit(self._submit_result, run_id, score)
+ self._executor.submit(self._submit_result, run_id, start_time, score)
- def _submit_result(self, run_id: uuid.UUID, score: Optional[int]) -> None:
+ def _submit_result(
+ self,
+ run_id: uuid.UUID,
+ start_time: datetime.datetime,
+ score: Optional[int],
+ ) -> None:
# trace_id will always be run_id here because the feedback is on the
root
# test run
- self.client.create_feedback(run_id, key="pass", score=score,
trace_id=run_id)
+ self.client.create_feedback(
+ run_id,
+ key="pass",
+ score=score,
+ trace_id=run_id,
+ session_id=self.experiment_id,
+ start_time=start_time,
+ )
def sync_example(
self,
@@ -726,10 +739,14 @@
self,
run_id: ID_TYPE,
feedback: Union[dict, list],
+ *,
+ start_time: datetime.datetime,
pytest_plugin: Any = None,
pytest_nodeid: Any = None,
**kwargs: Any,
):
+ kwargs["session_id"] = self.experiment_id
+ kwargs["start_time"] = start_time
feedback = feedback if isinstance(feedback, list) else [feedback]
for fb in feedback:
if pytest_plugin and pytest_nodeid:
@@ -819,6 +836,7 @@
self.pytest_nodeid = pytest_nodeid
self.inputs = inputs
self.reference_outputs = reference_outputs
+ self.run_start_time: Optional[datetime.datetime] = None
self._logged_reference_outputs: Optional[dict] = None
self._logged_outputs: Optional[dict] = None
@@ -832,11 +850,14 @@
self.log_reference_outputs(reference_outputs)
def submit_feedback(self, *args, **kwargs: Any):
+ if self.run_start_time is None:
+ raise ValueError("Cannot submit feedback before the test run has
started.")
self.test_suite._submit_feedback(
*args,
**{
**kwargs,
**dict(
+ start_time=self.run_start_time,
pytest_plugin=self.pytest_plugin,
pytest_nodeid=self.pytest_nodeid,
),
@@ -869,8 +890,13 @@
error: Optional[str] = None,
skipped: bool = False,
) -> None:
+ if self.run_start_time is None:
+ raise ValueError(
+ "Cannot submit a test result before the test run has started."
+ )
return self.test_suite.submit_result(
self.run_id,
+ self.run_start_time,
error=error,
skipped=skipped,
pytest_plugin=self.pytest_plugin,
@@ -1035,6 +1061,7 @@
exceptions_to_handle=(SkipException,),
_end_on_exit=False,
) as run_tree:
+ test_case.run_start_time = run_tree.start_time
try:
result = func(*test_args, **test_kwargs)
except SkipException as e:
@@ -1115,6 +1142,7 @@
exceptions_to_handle=(SkipException,),
_end_on_exit=False,
) as run_tree:
+ test_case.run_start_time = run_tree.start_time
try:
result = await func(*test_args, **test_kwargs)
except SkipException as e:
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/integration_tests/test_client.py
new/langsmith-0.10.10/tests/integration_tests/test_client.py
--- old/langsmith-0.10.9/tests/integration_tests/test_client.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/integration_tests/test_client.py
2020-02-02 01:00:00.000000000 +0100
@@ -4156,92 +4156,6 @@
assert insights_job.status in ["queued", "running", "success"]
-def test_feedback_formula_crud_flow(langchain_client: Client) -> None:
- dataset_name = f"feedback-formula-crud-{uuid7().hex}"
- feedback_key = f"overall-quality-{uuid7().hex[:8]}"
- initial_parts = [
- {"part_type": "weighted_key", "weight": 0.6, "key": "accuracy"},
- {"part_type": "weighted_key", "weight": 0.4, "key": "helpfulness"},
- ]
- updated_parts = [
- {"part_type": "weighted_key", "weight": 0.25, "key": "coverage"},
- {"part_type": "weighted_key", "weight": 0.75, "key": "relevance"},
- ]
-
- dataset = None
- feedback_formula_id = None
- try:
- dataset = langchain_client.create_dataset(dataset_name)
- created_formula = langchain_client.create_feedback_formula(
- feedback_key=feedback_key,
- aggregation_type="sum",
- formula_parts=initial_parts,
- dataset_id=dataset.id,
- )
- feedback_formula_id = created_formula.id
-
- assert created_formula.dataset_id == dataset.id
- assert created_formula.feedback_key == feedback_key
- assert [part.key for part in created_formula.formula_parts] == [
- part["key"] for part in initial_parts
- ]
-
- formulas =
list(langchain_client.list_feedback_formulas(dataset_id=dataset.id))
- assert any(formula.id == feedback_formula_id for formula in formulas)
-
- updated_feedback_key = f"{feedback_key}-updated"
- updated_formula = langchain_client.update_feedback_formula(
- feedback_formula_id,
- feedback_key=updated_feedback_key,
- aggregation_type="avg",
- formula_parts=updated_parts,
- )
- assert updated_formula.id == feedback_formula_id
- assert updated_formula.feedback_key == updated_feedback_key
- assert updated_formula.aggregation_type == "avg"
- assert [part.key for part in updated_formula.formula_parts] == [
- part["key"] for part in updated_parts
- ]
- assert [part.weight for part in updated_formula.formula_parts] == [
- part["weight"] for part in updated_parts
- ]
-
- fetched_formula = langchain_client.get_feedback_formula_by_id(
- feedback_formula_id
- )
- assert fetched_formula.feedback_key == updated_feedback_key
- assert fetched_formula.aggregation_type == "avg"
- assert [part.key for part in fetched_formula.formula_parts] == [
- part["key"] for part in updated_parts
- ]
-
- langchain_client.delete_feedback_formula(feedback_formula_id)
- deleted_formula_id = feedback_formula_id
- feedback_formula_id = None
-
- wait_for(
- lambda: (
- deleted_formula_id
- not in {
- formula.id
- for formula in langchain_client.list_feedback_formulas(
- dataset_id=dataset.id
- )
- }
- ),
- max_sleep_time=30,
- sleep_time=1,
- )
- finally:
- if feedback_formula_id is not None:
- try:
- langchain_client.delete_feedback_formula(feedback_formula_id)
- except Exception:
- pass
- if dataset is not None:
- safe_delete_dataset(langchain_client, dataset_id=dataset.id)
-
-
# ---------------------------------------------------------------------------
# v2 resource tests (runs.retrieve / runs.query via AsyncRunsResource)
# ---------------------------------------------------------------------------
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/unit_tests/evaluation/test_runner.py
new/langsmith-0.10.10/tests/unit_tests/evaluation/test_runner.py
--- old/langsmith-0.10.9/tests/unit_tests/evaluation/test_runner.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/evaluation/test_runner.py
2020-02-02 01:00:00.000000000 +0100
@@ -12,6 +12,7 @@
import uuid
from datetime import datetime, timezone
from threading import Lock
+from types import SimpleNamespace
from typing import Any, Callable, Dict, List, Tuple
from unittest import mock
from unittest.mock import MagicMock
@@ -21,11 +22,13 @@
from langsmith import Client, aevaluate, evaluate
from langsmith import schemas as ls_schemas
+from langsmith._internal._v2_migration_utils import _v2_run_to_schema
from langsmith.evaluation._runner import (
ComparativeExperimentResults,
_build_comparative_url,
_collect_evaluator_keys,
_get_target_args,
+ _load_traces_for_experiment,
)
from langsmith.evaluation.evaluator import (
DynamicRunEvaluator,
@@ -44,6 +47,7 @@
def __init__(self, ds_id, ds_name, ds_examples, tenant_id):
self.created_session = None
self.runs = {}
+ self.feedbacks = []
self.should_fail = False
self.ds_id = ds_id
self.ds_name = ds_name
@@ -116,6 +120,7 @@
}
return res
elif endpoint == "http://localhost:1984/feedback":
+ self.feedbacks.append(json.loads(kwargs["data"]))
response = MagicMock()
response.json.return_value = {}
return response
@@ -178,6 +183,79 @@
}
+def _fake_client_for_examples(
+ examples: list[dict[str, Any]], ds_id: str, ds_name: str
+) -> tuple[Client, FakeRequest]:
+ tenant_id = str(uuid.uuid4())
+ session = mock.Mock()
+ fake_request = FakeRequest(ds_id, ds_name, examples, tenant_id)
+ session.request = fake_request.request
+ client = Client(api_url="http://localhost:1984";, api_key="123",
session=session)
+ client._tenant_id = uuid.UUID(tenant_id)
+ return client, fake_request
+
+
+def test_evaluate_feedback_includes_experiment_session_id_and_run_start_time()
-> None:
+ example, example_payload = _create_example(0)
+ client, fake_request = _fake_client_for_examples(
+ [example_payload], str(example.dataset_id), "my-dataset"
+ )
+
+ def predict(inputs: dict) -> dict:
+ return {"output": inputs["in"] + 1}
+
+ def score(run, example):
+ return {"key": "quality", "score": 1}
+
+ results = evaluate(
+ predict,
+ data=[example],
+ evaluators=[score],
+ client=client,
+ blocking=True,
+ max_concurrency=0,
+ )
+ assert len(list(results)) == 1
+ _wait_until(lambda: len(fake_request.feedbacks) == 1)
+
+ feedback = fake_request.feedbacks[0]
+ _wait_until(lambda: feedback["run_id"] in fake_request.runs)
+ run = fake_request.runs[feedback["run_id"]]
+ assert feedback["session_id"] == fake_request.created_session["id"]
+ assert feedback["start_time"] == run["start_time"]
+
+
[email protected]
+async def test_aevaluate_feedback_includes_experiment_id_and_start_time() ->
None:
+ example, example_payload = _create_example(0)
+ client, fake_request = _fake_client_for_examples(
+ [example_payload], str(example.dataset_id), "my-dataset"
+ )
+
+ async def predict(inputs: dict) -> dict:
+ return {"output": inputs["in"] + 1}
+
+ async def score(run, example):
+ return {"key": "quality", "score": 1}
+
+ results = await aevaluate(
+ predict,
+ data=[example],
+ evaluators=[score],
+ client=client,
+ blocking=True,
+ max_concurrency=0,
+ )
+ assert len([row async for row in results]) == 1
+ _wait_until(lambda: len(fake_request.feedbacks) == 1)
+
+ feedback = fake_request.feedbacks[0]
+ _wait_until(lambda: feedback["run_id"] in fake_request.runs)
+ run = fake_request.runs[feedback["run_id"]]
+ assert feedback["session_id"] == fake_request.created_session["id"]
+ assert feedback["start_time"] == run["start_time"]
+
+
@pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or
higher")
@pytest.mark.parametrize("blocking", [False, True])
@pytest.mark.parametrize("as_runnable", [False, True])
@@ -1568,3 +1646,130 @@
legacy = ComparativeExperimentResults(results={}, examples={})
assert legacy.url is None
assert legacy.comparative_experiment is None
+
+
+def test_v2_run_to_schema_maps_generated_fields() -> None:
+ """Generated v2 Run (project_id, parent_run_ids) → ls_schemas.Run."""
+ run_id = uuid.uuid4()
+ trace_id = uuid.uuid4()
+ project_id = uuid.uuid4()
+ parent_id = uuid.uuid4()
+ example_id = uuid.uuid4()
+ start = datetime.now(timezone.utc)
+ # SimpleNamespace, not Mock: `name` is a reserved Mock kwarg.
+ generated = SimpleNamespace(
+ id=str(run_id),
+ name="root",
+ run_type="chain",
+ start_time=start,
+ end_time=None,
+ trace_id=str(trace_id),
+ project_id=str(project_id),
+ parent_run_ids=[str(uuid.uuid4()), str(parent_id)],
+ dotted_order="20240101T000000000000Z" + str(run_id),
+ reference_example_id=str(example_id),
+ inputs={"a": 1},
+ outputs={"b": 2},
+ error=None,
+ status="SUCCESS",
+ )
+
+ run = _v2_run_to_schema(generated)
+
+ assert run.id == run_id
+ assert run.trace_id == trace_id
+ # project_id → session_id, last parent_run_ids entry → parent_run_id
+ assert run.session_id == project_id
+ assert run.parent_run_id == parent_id
+ assert run.reference_example_id == example_id
+ assert run.outputs == {"b": 2}
+ assert run.status == "success"
+
+
+def test_v2_run_to_schema_handles_root_run() -> None:
+ """A root run (no parents) maps to parent_run_id=None."""
+ generated = SimpleNamespace(
+ id=str(uuid.uuid4()),
+ name="root",
+ run_type="chain",
+ start_time=datetime.now(timezone.utc),
+ end_time=None,
+ trace_id=str(uuid.uuid4()),
+ project_id=str(uuid.uuid4()),
+ parent_run_ids=[],
+ dotted_order=None,
+ reference_example_id=None,
+ inputs=None,
+ outputs=None,
+ error=None,
+ status=None,
+ )
+ run = _v2_run_to_schema(generated)
+ assert run.parent_run_id is None
+ assert run.inputs == {}
+
+
+def _make_generated_run() -> SimpleNamespace:
+ return SimpleNamespace(
+ id=str(uuid.uuid4()),
+ name="root",
+ run_type="chain",
+ start_time=datetime.now(timezone.utc),
+ end_time=None,
+ trace_id=str(uuid.uuid4()),
+ project_id=str(uuid.uuid4()),
+ parent_run_ids=[],
+ dotted_order=None,
+ reference_example_id=str(uuid.uuid4()),
+ inputs={},
+ outputs={},
+ error=None,
+ status="SUCCESS",
+ )
+
+
+def _experiment_session() -> ls_schemas.TracerSession:
+ return ls_schemas.TracerSession(
+ id=uuid.uuid4(),
+ name="exp",
+ tenant_id=uuid.uuid4(),
+ reference_dataset_id=uuid.uuid4(),
+ start_time=datetime(2020, 1, 1, tzinfo=timezone.utc),
+ )
+
+
+def test_load_traces_uses_v2_when_sdb_query_enabled() -> None:
+ """sdb_query_enabled → sync query_v2 (/v2/runs/query), never v1
list_runs."""
+ project = _experiment_session()
+ gen_runs = [_make_generated_run(), _make_generated_run()]
+
+ def _query_v2(**kwargs):
+ # Verify the query is scoped to this experiment, root-only, with a
window.
+ assert kwargs["project_ids"] == [str(project.id)]
+ assert kwargs["is_root"] is True
+ assert kwargs["min_start_time"] == project.start_time
+ assert "selects" in kwargs
+ return iter(gen_runs) # sync paginator auto-iterates with a plain
for-loop
+
+ client = mock.Mock()
+ client.info.instance_flags = {"sdb_query_enabled": True}
+ client._get_langsmith_api_sync.return_value.runs.query_v2 = _query_v2
+
+ runs = _load_traces_for_experiment(project, client, load_nested=False)
+
+ assert len(runs) == 2
+ assert all(isinstance(r, ls_schemas.Run) for r in runs)
+ client.list_runs.assert_not_called()
+
+
+def test_load_traces_uses_v1_when_sdb_query_disabled() -> None:
+ """Without the flag, keep the existing v1 client.list_runs path
unchanged."""
+ project = _experiment_session()
+ client = mock.Mock()
+ client.info.instance_flags = {} # sdb_query_enabled absent
+ client.list_runs.return_value = iter([])
+
+ _load_traces_for_experiment(project, client, load_nested=False)
+
+ client.list_runs.assert_called_once_with(project_id=project.id,
is_root=True)
+ client._get_langsmith_api_sync.assert_not_called()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/unit_tests/sandbox/test_async_client.py
new/langsmith-0.10.10/tests/unit_tests/sandbox/test_async_client.py
--- old/langsmith-0.10.9/tests/unit_tests/sandbox/test_async_client.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/sandbox/test_async_client.py
2020-02-02 01:00:00.000000000 +0100
@@ -116,7 +116,8 @@
assert client._http.headers.get("X-Service-Key") == "svc-jwt"
assert client._http.headers.get("X-Api-Key") == "api-key"
assert client._default_headers == {"X-Service-Key": "svc-jwt"}
- assert client._ws_default_headers(None) == {"X-Service-Key":
"svc-jwt"}
+ # merge_headers normalizes names to lowercase.
+ assert client._ws_default_headers(None) == {"x-service-key":
"svc-jwt"}
await client.aclose()
async def test_max_retries_default(self):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/unit_tests/sandbox/test_client.py
new/langsmith-0.10.10/tests/unit_tests/sandbox/test_client.py
--- old/langsmith-0.10.9/tests/unit_tests/sandbox/test_client.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/sandbox/test_client.py
2020-02-02 01:00:00.000000000 +0100
@@ -133,13 +133,14 @@
api_key="api-key",
headers={"X-Service-Key": "svc-jwt"},
)
- assert client._ws_default_headers(None) == {"X-Service-Key": "svc-jwt"}
+ # merge_headers normalizes names to lowercase.
+ assert client._ws_default_headers(None) == {"x-service-key": "svc-jwt"}
assert client._ws_default_headers({"X-Test": "v"}) == {
- "X-Service-Key": "svc-jwt",
- "X-Test": "v",
+ "x-service-key": "svc-jwt",
+ "x-test": "v",
}
assert client._ws_default_headers({"X-Service-Key": "override"}) == {
- "X-Service-Key": "override"
+ "x-service-key": "override"
}
client.close()
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/unit_tests/sandbox/test_helpers.py
new/langsmith-0.10.10/tests/unit_tests/sandbox/test_helpers.py
--- old/langsmith-0.10.9/tests/unit_tests/sandbox/test_helpers.py
1970-01-01 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/sandbox/test_helpers.py
2020-02-02 01:00:00.000000000 +0100
@@ -0,0 +1,39 @@
+"""Unit tests for langsmith.sandbox._helpers."""
+
+import httpx
+
+from langsmith.sandbox._helpers import merge_headers
+
+
+def test_merge_headers_override_wins() -> None:
+ # Names are normalized to lowercase and the override wins.
+ merged = merge_headers({"X-Service-Key": "base"}, {"X-Service-Key":
"override"})
+ assert merged == {"x-service-key": "override"}
+
+
+def test_merge_headers_preserves_non_overridden_base() -> None:
+ merged = merge_headers({"a": "1", "b": "2"}, {"b": "3"})
+ assert merged == {"a": "1", "b": "3"}
+
+
+def test_merge_headers_none_inputs() -> None:
+ assert merge_headers(None, None) == {}
+ assert merge_headers({"a": "1"}, None) == {"a": "1"}
+ assert merge_headers(None, {"a": "1"}) == {"a": "1"}
+
+
+def test_merge_headers_override_replaces_across_casing() -> None:
+ # httpx.Headers normalizes names to lowercase; a Title-Case override must
+ # still replace the base rather than produce a second, duplicate header.
+ base = httpx.Headers({"X-Service-Key": "base", "X-Api-Key": ""})
+ merged = merge_headers(base, {"X-Service-Key": "override"})
+
+ assert [v for k, v in merged.items() if k.lower() == "x-service-key"] == [
+ "override"
+ ]
+ # And only one X-Service-Key actually goes on the wire.
+ request = httpx.Request("GET", "https://example.com";, headers=merged)
+ on_wire = [
+ v.decode() for k, v in request.headers.raw if k.lower() ==
b"x-service-key"
+ ]
+ assert on_wire == ["override"]
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/langsmith-0.10.9/tests/unit_tests/sandbox/test_ws_execute.py
new/langsmith-0.10.10/tests/unit_tests/sandbox/test_ws_execute.py
--- old/langsmith-0.10.9/tests/unit_tests/sandbox/test_ws_execute.py
2020-02-02 01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/sandbox/test_ws_execute.py
2020-02-02 01:00:00.000000000 +0100
@@ -74,8 +74,9 @@
class TestBuildAuthHeaders:
def test_builds_header(self):
+ # Header names are normalized to lowercase.
headers = _build_auth_headers("my-key")
- assert headers == {"X-Api-Key": "my-key"}
+ assert headers == {"x-api-key": "my-key"}
def test_none_key_returns_empty(self):
headers = _build_auth_headers(None)
@@ -87,8 +88,8 @@
{"X-Api-Key": "override-key", "X-Test-Header": "ws-value"},
)
assert headers == {
- "X-Api-Key": "override-key",
- "X-Test-Header": "ws-value",
+ "x-api-key": "override-key",
+ "x-test-header": "ws-value",
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/tests/unit_tests/test_client.py
new/langsmith-0.10.10/tests/unit_tests/test_client.py
--- old/langsmith-0.10.9/tests/unit_tests/test_client.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/test_client.py 2020-02-02
01:00:00.000000000 +0100
@@ -6734,6 +6734,13 @@
)
+def test_get_run_stats_requires_project() -> None:
+ """get_run_stats must raise ValueError when no project is specified."""
+ client = Client(api_key="test", api_url="http://localhost";)
+ with pytest.raises(ValueError, match="project_names or project_ids"):
+ client.get_run_stats()
+
+
def test_removed_sdk_methods_absent() -> None:
"""Verify de-publicized methods were removed from the generated SDK in PR
#28358."""
from langsmith._openapi_client.resources.datasets.datasets import
DatasetsResource
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/langsmith-0.10.9/tests/unit_tests/test_testing.py
new/langsmith-0.10.10/tests/unit_tests/test_testing.py
--- old/langsmith-0.10.9/tests/unit_tests/test_testing.py 2020-02-02
01:00:00.000000000 +0100
+++ new/langsmith-0.10.10/tests/unit_tests/test_testing.py 2020-02-02
01:00:00.000000000 +0100
@@ -1,8 +1,10 @@
+import datetime
import uuid
from unittest.mock import MagicMock
from langsmith.testing._internal import (
_get_example_id,
+ _LangSmithTestSuite,
_serde_example_values,
_TestCase,
)
@@ -76,3 +78,29 @@
actual_id = _get_example_id(dataset_id, test_case.inputs or {})
assert actual_id == expected_id
assert actual_id != fixture_id
+
+
+def test_pytest_feedback_includes_experiment_id_and_run_start_time():
+ experiment_id = uuid.uuid4()
+ run_id = uuid.uuid4()
+ start_time = datetime.datetime.now(datetime.timezone.utc)
+ test_suite = object.__new__(_LangSmithTestSuite)
+ test_suite.client = MagicMock()
+ test_suite._experiment = MagicMock(id=experiment_id)
+ test_suite._executor = MagicMock()
+ test_suite._executor.submit.side_effect = lambda fn, *args, **kwargs: fn(
+ *args, **kwargs
+ )
+
+ test_suite._submit_result(run_id, start_time, 1)
+ test_suite._submit_feedback(
+ run_id,
+ {"key": "quality", "score": 0.75},
+ start_time=start_time,
+ )
+
+ assert test_suite.client.create_feedback.call_count == 2
+ for call in test_suite.client.create_feedback.call_args_list:
+ assert call.args[0] == run_id
+ assert call.kwargs["session_id"] == experiment_id
+ assert call.kwargs["start_time"] == start_time
