This is an automated email from the ASF dual-hosted git repository.
tlopex pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm-ffi.git
The following commit(s) were added to refs/heads/main by this push:
new eed79b04 [STUBGEN] Classify native layouts from reflected byte facts
(#730)
eed79b04 is described below
commit eed79b047b56afaccad87a5d29b2ead2f5aad188
Author: Linzhang Li <[email protected]>
AuthorDate: Thu Sep 3 19:50:43 2026 -0400
[STUBGEN] Classify native layouts from reflected byte facts (#730)
# [STUBGEN] Classify native layouts from reflected byte facts
## Summary
Add a language-neutral classifier that decides, for every registered
object type, whether its native layout can be reproduced from reflection
(`complete`) or not (`opaque`), with the byte evidence behind each
verdict. `tvm-ffi-stubgen --coverage-out <json>` writes the verdicts as
a report. No generator is attached.
## Motivation
A native-struct backend can hold an object by value only when it can
reproduce the C++ layout byte for byte. The registry already publishes
what that needs: each type's own `sizeof` (#720) and every field's size,
alignment and offset. A wrong `complete` is a struct with the wrong
layout, a wrong `opaque` only costs a C ABI call per field, so the
classifier deserves a review of its own before any generator builds on
it.
## Changes
- `stub/layout.py` (new). One criterion: the reflected fields must fill
`[parent.total_size, total_size)` exactly, allowing only the gaps
alignment forces. Two prerequisites: the type has metadata of its own
(`layout-unknown`) and its parent is complete (`parent-opaque`).
Unexplained bytes are `uncovered-bytes`; a field starting before the
cursor is `field-overlap`. Polymorphism is not a rule: a vptr makes the
fields fall short of `sizeof` on its own. Two target-language rules are
injection points: `field_renderable` and `forced_opaque`, the latter
only demoting a type that would otherwise be complete.
- `stub/cli.py`, `stub/utils.py`: `--coverage-out <json>`, usable
without PATH arguments. The report fixes the language-neutral keys; a
generator adds its own under a per-type `"target"` key.
- `testing.cc`: `TestCxxClassHiddenField` (an unreflected member) and
`TestCxxClassPolymorphic` (a vptr), so both failure paths run in CI.
## Testing
`tests/python/test_stub_layout.py`: every verdict path on synthetic byte
facts, then the real registry. The `TestCxxClass*` chain is complete
with padding `[36, 40)`, `[52, 56)`, `[73, 80)`;
`TestCxxClassHiddenField` reports `[32, 40)` uncovered;
`TestCxxClassPolymorphic` reports `sizeof` 40 against fields ending at
32; `ffi.Function` is `layout-unknown`; `ffi.Module` is opaque like the
polymorphic fixture. The CLI writes the report with `--coverage-out`
alone.
---------
Signed-off-by: yuchuan <[email protected]>
---
python/tvm_ffi/stub/cli.py | 24 ++-
python/tvm_ffi/stub/layout.py | 361 ++++++++++++++++++++++++++++++++++++++
python/tvm_ffi/stub/utils.py | 5 +-
src/ffi/testing/testing.cc | 29 ++++
tests/python/test_stub_layout.py | 362 +++++++++++++++++++++++++++++++++++++++
5 files changed, 779 insertions(+), 2 deletions(-)
diff --git a/python/tvm_ffi/stub/cli.py b/python/tvm_ffi/stub/cli.py
index 3dfc8151..a5e0ba39 100644
--- a/python/tvm_ffi/stub/cli.py
+++ b/python/tvm_ffi/stub/cli.py
@@ -29,6 +29,7 @@ from typing import TYPE_CHECKING
from . import consts as C
from .file_utils import FileInfo, collect_files, syntax_for
from .generator import generator_names, get_generator
+from .layout import classify, write_coverage_report
from .lib_state import (
collect_global_funcs,
collect_type_keys,
@@ -111,6 +112,15 @@ def __main__() -> int:
if opt.init and generated_prefixes:
assert init_path is not None
generator.finalize_init(init_path, generated_prefixes)
+
+ # Write the native-layout coverage report, if requested.
+ if opt.coverage_out is not None:
+ infos = {
+ type_key: object_info_from_type_key(type_key)
+ for type_keys in collect_type_keys().values()
+ for type_key in type_keys
+ }
+ write_coverage_report(Path(opt.coverage_out), classify(infos))
del dlls
return 0
@@ -380,6 +390,17 @@ def _parse_args() -> Options:
"without modifying any files."
),
)
+ parser.add_argument(
+ "--coverage-out",
+ type=str,
+ default=None,
+ metavar="JSON",
+ help=(
+ "Write a JSON report classifying every registered object type by
whether "
+ "its native memory layout can be reproduced from reflection
(complete) or "
+ "not (opaque), with the byte evidence. May be used without PATH
arguments."
+ ),
+ )
args = parser.parse_args()
init_flags = [args.init_pypkg, args.init_lib, args.init_prefix]
@@ -393,7 +414,7 @@ def _parse_args() -> Options:
prefix=args.init_prefix,
)
- if not args.files:
+ if not args.files and args.coverage_out is None:
parser.print_help()
sys.exit(1)
@@ -406,6 +427,7 @@ def _parse_args() -> Options:
verbose=args.verbose,
dry_run=args.dry_run,
target=args.target,
+ coverage_out=args.coverage_out,
)
diff --git a/python/tvm_ffi/stub/layout.py b/python/tvm_ffi/stub/layout.py
new file mode 100644
index 00000000..798676a4
--- /dev/null
+++ b/python/tvm_ffi/stub/layout.py
@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Native-layout classification for ``tvm-ffi-stubgen``.
+
+A generator for a language with native structs can hold a reflected object *by
+value* only when it can reproduce the object's memory layout byte for byte. The
+registry publishes what the compiler computed: each type's own ``sizeof``
+(:attr:`.ObjectInfo.total_size`) and every reflected field's size, alignment
and
+offset. One criterion decides the question:
+
+ The reflected fields must fill ``[parent.total_size, total_size)`` exactly,
+ allowing only the gaps alignment forces.
+
+The cursor starts at the parent's size; each field, by ascending offset, must
+start at ``align_up(cursor, field.alignment)``; at the end
+``align_up(cursor, alignment)`` must equal ``total_size``, where ``alignment``
is
+the largest alignment along the chain, the header's included. Two facts must
+exist before the criterion can be evaluated: the type needs metadata of its own
+(a type without an ``ObjectDef`` inherits its parent's entry, whose size says
+nothing about it) and its parent must itself be complete. Nothing else is a
+rule: a class with a vptr, for instance, reports offsets relative to the
+``TVMFFIObject`` header while its ``sizeof`` is absolute, so its fields can
never
+fill the region.
+
+``opaque`` is a normal, final verdict: a generator still emits a wrapper and
+reaches the fields through the C ABI. Two target-language rules are injected by
+the caller instead of living here: a field-renderability predicate, and a set
of
+semantic vetoes for types that must never be allocated outside their runtime.
+
+The coverage report fixes the language-neutral keys of each entry; a generator
+adds its own facts under a per-type ``"target"`` key.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+from typing import TYPE_CHECKING, Any, Callable, Literal
+
+from typing_extensions import TypeAlias
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping
+ from collections.abc import Set as AbstractSet
+ from pathlib import Path
+
+ from .utils import NamedTypeSchema, ObjectInfo
+
+#: Alignment of the ``TVMFFIObject`` header (a 64-bit reference count); no
object is aligned less.
+OBJECT_HEADER_ALIGNMENT = 8
+
+OpaqueReason: TypeAlias = Literal[
+ "layout-unknown",
+ "parent-opaque",
+ "field-overlap",
+ "uncovered-bytes",
+ "unrenderable-field",
+ "by-directive",
+]
+"""Why a type is opaque.
+
+- ``layout-unknown``: no metadata of its own, or a field without byte facts.
+- ``parent-opaque``: the parent is opaque, so the region to fill has no
+ trustworthy start.
+- ``field-overlap``: a field starts before the cursor (e.g. inside the parent's
+ tail padding, which the Itanium ABI may reuse) or ends past ``total_size``.
+- ``uncovered-bytes``: a gap alignment does not explain (an unreflected member,
+ a vptr, ...).
+- ``unrenderable-field``: the caller's predicate rejected a field.
+- ``by-directive``: the caller vetoed a type whose layout is reproducible.
+"""
+
+
+def align_up(value: int, alignment: int) -> int:
+ """Round ``value`` up to the next multiple of ``alignment``."""
+ return (value + alignment - 1) // alignment * alignment
+
+
[email protected](frozen=True)
+class ByteRange:
+ """A half-open byte interval ``[start, end)`` inside an object."""
+
+ start: int
+ end: int
+
+ def __str__(self) -> str:
+ return f"[{self.start}, {self.end})"
+
+ def to_json_obj(self) -> list[int]:
+ """Render as ``[start, end]``."""
+ return [self.start, self.end]
+
+
[email protected](frozen=True)
+class FieldBytes:
+ """The byte facts of one reflected field."""
+
+ name: str
+ offset: int
+ size: int
+ alignment: int
+
+ @property
+ def end(self) -> int:
+ """One past the last byte of the field."""
+ return self.offset + self.size
+
+ def to_json_obj(self) -> dict[str, Any]:
+ """Render as a JSON object."""
+ return dataclasses.asdict(self)
+
+
[email protected]
+class Verdict:
+ """The layout verdict of one type, with the byte evidence behind it."""
+
+ type_key: str
+ verdict: Literal["complete", "opaque"]
+ reason: OpaqueReason | None
+ """``None`` for a complete type; otherwise which rule demoted it."""
+ detail: str
+ """Human-readable explanation, quoting the byte evidence."""
+ parent_type_key: str | None
+ ancestors: list[str]
+ total_size: int | None
+ is_final: bool | None
+ alignment: int | None = None
+ """Natural alignment of the object, derived along the chain (drives the
tail check)."""
+ own_bytes: ByteRange | None = None
+ """The region this type's own fields must fill: ``[parent.total_size,
total_size)``."""
+ fields: list[FieldBytes] = dataclasses.field(default_factory=list)
+ """This type's own reflected fields by ascending offset; empty when the
layout is unknown."""
+ uncovered: list[ByteRange] = dataclasses.field(default_factory=list)
+ """Bytes inside ``own_bytes`` that neither a field nor an alignment gap
accounts for."""
+ padding: list[ByteRange] = dataclasses.field(default_factory=list)
+ """Alignment-forced gaps, recorded so a reviewer can check them against
the C++ declaration."""
+
+ @property
+ def is_complete(self) -> bool:
+ """Whether the native layout is reproducible from the reflected
fields."""
+ return self.verdict == "complete"
+
+ def to_json_obj(self) -> dict[str, Any]:
+ """Render the language-neutral report entry for this type."""
+ return {
+ "verdict": self.verdict,
+ "reason": self.reason,
+ "detail": self.detail,
+ "parent": self.parent_type_key,
+ "ancestors": list(self.ancestors),
+ "total_size": self.total_size,
+ "is_final": self.is_final,
+ "own_bytes": None if self.own_bytes is None else
self.own_bytes.to_json_obj(),
+ "fields": [f.to_json_obj() for f in self.fields],
+ "uncovered": [r.to_json_obj() for r in self.uncovered],
+ "padding": [r.to_json_obj() for r in self.padding],
+ }
+
+
+def classify(
+ infos: Mapping[str, ObjectInfo],
+ *,
+ forced_opaque: AbstractSet[str] = frozenset(),
+ field_renderable: Callable[[NamedTypeSchema], bool] | None = None,
+) -> dict[str, Verdict]:
+ """Classify every type in ``infos``, parents before children.
+
+ Parameters
+ ----------
+ infos
+ The types to classify, keyed by type key, in any order. Every ancestor
+ of a type must be present as well.
+ forced_opaque
+ Type keys vetoed by the caller (semantic blockers such as interned
+ identities). The veto only demotes a type that would otherwise be
+ complete; a type that is opaque for a layout reason keeps that reason.
+ field_renderable
+ Predicate deciding whether a reflected field has a native mirror in the
+ target language. A rejected field demotes its type to opaque with
+ reason ``unrenderable-field``. ``None`` accepts every field.
+
+ """
+ verdicts: dict[str, Verdict] = {}
+
+ def _classify(type_key: str) -> Verdict:
+ if type_key in verdicts:
+ return verdicts[type_key]
+ if type_key not in infos:
+ raise KeyError(f"Ancestor {type_key!r} is not among the types to
classify")
+ info = infos[type_key]
+ parent = None if info.parent_type_key is None else
_classify(info.parent_type_key)
+ verdicts[type_key] = _classify_one(info, parent, forced_opaque,
field_renderable)
+ return verdicts[type_key]
+
+ for type_key in infos:
+ _classify(type_key)
+ return verdicts
+
+
+def _classify_one(
+ info: ObjectInfo,
+ parent: Verdict | None,
+ forced_opaque: AbstractSet[str],
+ field_renderable: Callable[[NamedTypeSchema], bool] | None,
+) -> Verdict:
+ assert info.type_key is not None, "cannot classify an ObjectInfo without a
type key"
+ verdict = Verdict(
+ type_key=info.type_key,
+ verdict="opaque",
+ reason=None,
+ detail="",
+ parent_type_key=info.parent_type_key,
+ ancestors=list(info.ancestors),
+ total_size=info.total_size,
+ is_final=info.is_final,
+ )
+ outcome = _prove_layout(info, parent, verdict)
+ if outcome is None:
+ outcome = _apply_target_rules(info, forced_opaque, field_renderable)
+ if outcome is not None:
+ verdict.reason, verdict.detail = outcome
+ return verdict
+ verdict.verdict = "complete"
+ if parent is None:
+ verdict.detail = f"object header [0, {info.total_size}): owned by the
C ABI"
+ else:
+ verdict.detail = f"reflected fields fill {verdict.own_bytes} exactly"
+ if verdict.padding:
+ gaps = ", ".join(str(r) for r in verdict.padding)
+ verdict.detail += f" (alignment padding {gaps})"
+ return verdict
+
+
+def _prove_layout(
+ info: ObjectInfo, parent: Verdict | None, verdict: Verdict
+) -> tuple[OpaqueReason, str] | None:
+ """Check the prerequisites and the fill criterion, recording the evidence
on ``verdict``.
+
+ Returns ``None`` when the layout is reproducible, else the reason it is
not.
+ """
+ # Nothing to check without the type's own size and every field's byte
facts.
+ if info.total_size is None:
+ return "layout-unknown", "no metadata of its own: total_size is
unknown"
+ fields = _field_bytes(info)
+ if isinstance(fields, str):
+ return "layout-unknown", fields
+ verdict.fields = fields
+
+ # The parent's size is where this type's own bytes start, so the parent
must be complete.
+ if parent is None:
+ # The root is the `TVMFFIObject` header: C ABI bytes, nothing to fill.
+ verdict.alignment = OBJECT_HEADER_ALIGNMENT
+ verdict.own_bytes = ByteRange(info.total_size, info.total_size)
+ return None
+ if not parent.is_complete:
+ return "parent-opaque", f"parent {parent.type_key!r} is opaque
({parent.reason})"
+ assert parent.total_size is not None and parent.alignment is not None
+
+ # Fields must fill [parent.total_size, total_size) exactly.
+ verdict.own_bytes = ByteRange(parent.total_size, info.total_size)
+ verdict.alignment = max([parent.alignment, *(f.alignment for f in fields)])
+ return _fill(verdict)
+
+
+def _field_bytes(info: ObjectInfo) -> list[FieldBytes] | str:
+ """Return the type's own fields by ascending offset, or why their bytes
are unknown."""
+ fields: list[FieldBytes] = []
+ for f in info.fields:
+ if f.size is None or f.alignment is None or f.offset is None:
+ return f"field {f.name!r} carries no native layout facts"
+ fields.append(FieldBytes(name=f.name, offset=f.offset, size=f.size,
alignment=f.alignment))
+ return sorted(fields, key=lambda f: f.offset)
+
+
+def _fill(verdict: Verdict) -> tuple[OpaqueReason, str] | None:
+ """Walk ``verdict.fields`` over ``verdict.own_bytes``, sorting every gap
into padding or hole.
+
+ Returns why the fill fails (the first overlap, else the uncovered bytes),
or ``None``.
+ """
+ assert verdict.own_bytes is not None and verdict.alignment is not None
+ end = verdict.own_bytes.end
+ overlap: str | None = None
+ cursor = verdict.own_bytes.start
+ for field in verdict.fields:
+ if field.offset < cursor:
+ where = f"[{field.offset}, {field.end}) starts before byte
{cursor}"
+ overlap = overlap or f"field {field.name!r} at {where}"
+ cursor = max(cursor, field.end)
+ continue
+ _record_gap(verdict, cursor, align_up(cursor, field.alignment),
field.offset)
+ cursor = field.end
+ if cursor > end:
+ overlap = overlap or f"fields extend to byte {cursor}, past total_size
{end}"
+ else:
+ _record_gap(verdict, cursor, align_up(cursor, verdict.alignment), end)
+ if overlap is not None:
+ return "field-overlap", overlap
+ if verdict.uncovered:
+ ranges = ", ".join(str(r) for r in verdict.uncovered)
+ return "uncovered-bytes", (
+ f"bytes {ranges} of {verdict.own_bytes} are not accounted for by
reflected fields"
+ )
+ return None
+
+
+def _record_gap(verdict: Verdict, cursor: int, expected: int, actual: int) ->
None:
+ """Classify the gap between ``cursor`` and the next boundary ``actual``.
+
+ ``expected`` is where alignment alone would put that boundary: a gap that
+ ends exactly there is padding, any other gap is a hole.
+ """
+ if expected != actual:
+ verdict.uncovered.append(ByteRange(cursor, actual))
+ elif expected != cursor:
+ verdict.padding.append(ByteRange(cursor, expected))
+
+
+def _apply_target_rules(
+ info: ObjectInfo,
+ forced_opaque: AbstractSet[str],
+ field_renderable: Callable[[NamedTypeSchema], bool] | None,
+) -> tuple[OpaqueReason, str] | None:
+ """Apply the caller-injected, target-language rules to a type whose layout
is proven."""
+ if field_renderable is not None:
+ for field in info.fields:
+ if not field_renderable(field):
+ return "unrenderable-field", (
+ f"field {field.name!r} ({field.repr()}) has no native
mirror"
+ )
+ if info.type_key in forced_opaque:
+ return "by-directive", "vetoed by directive although the layout is
reproducible"
+ return None
+
+
+def coverage_report(verdicts: Mapping[str, Verdict]) -> dict[str, Any]:
+ """Build the JSON-serialisable coverage report: one entry per type key,
sorted.
+
+ Each entry holds the language-neutral keys rendered by
+ :meth:`Verdict.to_json_obj`; a generator may add a ``"target"`` key with
+ its own facts.
+ """
+ return {key: verdicts[key].to_json_obj() for key in sorted(verdicts)}
+
+
+def write_coverage_report(path: Path, verdicts: Mapping[str, Verdict]) -> None:
+ """Write :func:`coverage_report` to ``path`` as indented JSON."""
+ path.write_text(json.dumps(coverage_report(verdicts), indent=2) + "\n",
encoding="utf-8")
diff --git a/python/tvm_ffi/stub/utils.py b/python/tvm_ffi/stub/utils.py
index a053758e..7f5ea625 100644
--- a/python/tvm_ffi/stub/utils.py
+++ b/python/tvm_ffi/stub/utils.py
@@ -85,6 +85,8 @@ class Options:
dry_run: bool = False
target: str = "python"
"""Code generator target to use."""
+ coverage_out: str | None = None
+ """Path of the native-layout coverage report (JSON) to write, if
requested."""
@dataclasses.dataclass(init=False)
@@ -248,8 +250,9 @@ class ObjectInfo:
)
)
+ # `fields` is None while a `py_class` registration is still pending.
return ObjectInfo(
- fields=[NamedTypeSchema.from_type_field(field) for field in
type_info.fields],
+ fields=[NamedTypeSchema.from_type_field(field) for field in
type_info.fields or []],
methods=[
FuncInfo(
schema=NamedTypeSchema(
diff --git a/src/ffi/testing/testing.cc b/src/ffi/testing/testing.cc
index 21b87b46..742ca9d0 100644
--- a/src/ffi/testing/testing.cc
+++ b/src/ffi/testing/testing.cc
@@ -206,6 +206,29 @@ class TestCxxClassDerivedDerived : public
TestCxxClassDerived {
TestCxxClassDerived);
};
+// Fixtures for the stub layout classifier: an unreflected member, and a vptr.
+class TestCxxClassHiddenField : public Object {
+ public:
+ int64_t v_i64 = 0;
+ int64_t hidden_i64 = 0; // deliberately not reflected
+ int32_t v_i32 = 0;
+
+ static constexpr bool _type_mutable = true;
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("testing.TestCxxClassHiddenField",
TestCxxClassHiddenField,
+ Object);
+};
+
+class TestCxxClassPolymorphic : public Object {
+ public:
+ int64_t v_i64 = 0;
+
+ virtual ~TestCxxClassPolymorphic() = default;
+
+ static constexpr bool _type_mutable = true;
+ TVM_FFI_DECLARE_OBJECT_INFO_FINAL("testing.TestCxxClassPolymorphic",
TestCxxClassPolymorphic,
+ Object);
+};
+
class TestCxxEnumHolderObj : public Object {
public:
TestCxxIntEnum priority;
@@ -498,6 +521,12 @@ TVM_FFI_STATIC_INIT_BLOCK() {
.def_rw("v_str", &TestCxxClassDerivedDerived::v_str,
refl::default_value(String("default")))
.def_rw("v_bool", &TestCxxClassDerivedDerived::v_bool);
+ refl::ObjectDef<TestCxxClassHiddenField>()
+ .def_rw("v_i64", &TestCxxClassHiddenField::v_i64)
+ .def_rw("v_i32", &TestCxxClassHiddenField::v_i32);
+
+ refl::ObjectDef<TestCxxClassPolymorphic>().def_rw("v_i64",
&TestCxxClassPolymorphic::v_i64);
+
refl::ObjectDef<TestCxxEnumHolderObj>()
.def_rw("priority", &TestCxxEnumHolderObj::priority)
.def_rw("opcode", &TestCxxEnumHolderObj::opcode);
diff --git a/tests/python/test_stub_layout.py b/tests/python/test_stub_layout.py
new file mode 100644
index 00000000..e0a6edcb
--- /dev/null
+++ b/tests/python/test_stub_layout.py
@@ -0,0 +1,362 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Tests for the native-layout classifier behind ``tvm-ffi-stubgen
--coverage-out``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+import tvm_ffi.stub.cli as stub_cli
+import tvm_ffi.testing # noqa: F401 (loads the `testing.*` fixture types)
+from tvm_ffi import core
+from tvm_ffi.core import TypeSchema
+from tvm_ffi.stub.layout import (
+ OBJECT_HEADER_ALIGNMENT,
+ ByteRange,
+ FieldBytes,
+ Verdict,
+ classify,
+ coverage_report,
+ write_coverage_report,
+)
+from tvm_ffi.stub.lib_state import collect_type_keys, object_info_from_type_key
+from tvm_ffi.stub.utils import NamedTypeSchema, ObjectInfo
+
+HEADER = 24 # sizeof(TVMFFIObject)
+
+
+# ---------------------------------------------------------------------------
+# Synthetic fixtures: the criterion is a pure function of the byte facts.
+# ---------------------------------------------------------------------------
+
+
+def _field(
+ name: str, offset: int | None, size: int | None, alignment: int | None =
None
+) -> NamedTypeSchema:
+ if alignment is None:
+ alignment = size
+ return NamedTypeSchema(name, TypeSchema("int"), size=size,
alignment=alignment, offset=offset)
+
+
+_ANCESTORS: dict[str, list[str]] = {}
+
+
+def _info(
+ type_key: str,
+ *,
+ parent: str | None = None,
+ total_size: int | None,
+ fields: tuple[NamedTypeSchema, ...] = (),
+ is_final: bool | None = None,
+) -> ObjectInfo:
+ ancestors = [] if parent is None else [*_ANCESTORS[parent], parent]
+ _ANCESTORS[type_key] = ancestors
+ return ObjectInfo(
+ fields=list(fields),
+ methods=[],
+ type_key=type_key,
+ parent_type_key=parent,
+ ancestors=ancestors,
+ total_size=total_size,
+ is_final=is_final,
+ )
+
+
+ROOT = _info("ffi.Object", total_size=HEADER)
+# int64 @24, int32 @32, tail padding [36, 40): the shape of
`testing.TestCxxClassBase`.
+BASE = _info(
+ "t.Base",
+ parent="ffi.Object",
+ total_size=40,
+ fields=(_field("v_i64", 24, 8), _field("v_i32", 32, 4)),
+ is_final=False,
+)
+
+
+def _ranges(ranges: list[ByteRange]) -> list[tuple[int, int]]:
+ return [(r.start, r.end) for r in ranges]
+
+
+def test_root_is_the_object_header() -> None:
+ """The root of the hierarchy is the C ABI header: complete, nothing to
fill."""
+ verdict = classify({"ffi.Object": ROOT})["ffi.Object"]
+ assert verdict.is_complete
+ assert verdict.reason is None
+ assert verdict.own_bytes == ByteRange(HEADER, HEADER)
+ assert verdict.alignment == OBJECT_HEADER_ALIGNMENT
+ assert "object header" in verdict.detail
+
+
+def test_complete_with_alignment_padding() -> None:
+ verdict = classify({"ffi.Object": ROOT, "t.Base": BASE})["t.Base"]
+ assert verdict.is_complete
+ assert verdict.reason is None
+ assert verdict.own_bytes == ByteRange(HEADER, 40)
+ assert verdict.alignment == 8
+ assert verdict.fields == [FieldBytes("v_i64", 24, 8, 8),
FieldBytes("v_i32", 32, 4, 4)]
+ assert _ranges(verdict.padding) == [(36, 40)]
+ assert verdict.uncovered == []
+ assert "[24, 40)" in verdict.detail and "[36, 40)" in verdict.detail
+
+
+def test_fields_are_visited_by_offset_not_declaration_order() -> None:
+ info = _info(
+ "t.Shuffled",
+ parent="ffi.Object",
+ total_size=40,
+ fields=(_field("late", 32, 8), _field("early", 24, 8)),
+ )
+ verdict = classify({"ffi.Object": ROOT, "t.Shuffled": info})["t.Shuffled"]
+ assert verdict.is_complete
+ assert [f.name for f in verdict.fields] == ["early", "late"]
+
+
+def test_uncovered_gap_in_the_middle() -> None:
+ """A member the registry never saw: [32, 40) is neither a field nor
alignment."""
+ info = _info(
+ "t.Hidden",
+ parent="ffi.Object",
+ total_size=48,
+ fields=(_field("v_i64", 24, 8), _field("v_i32", 40, 4)),
+ )
+ verdict = classify({"ffi.Object": ROOT, "t.Hidden": info})["t.Hidden"]
+ assert verdict.verdict == "opaque"
+ assert verdict.reason == "uncovered-bytes"
+ assert _ranges(verdict.uncovered) == [(32, 40)]
+ assert _ranges(verdict.padding) == [(44, 48)] # the tail is still
explained by alignment
+ assert "[32, 40)" in verdict.detail
+
+
+def test_uncovered_tail() -> None:
+ """The polymorphic shape: offsets are header-relative, `sizeof` is
absolute."""
+ info = _info("t.Poly", parent="ffi.Object", total_size=40,
fields=(_field("v_i64", 24, 8),))
+ verdict = classify({"ffi.Object": ROOT, "t.Poly": info})["t.Poly"]
+ assert verdict.reason == "uncovered-bytes"
+ assert _ranges(verdict.uncovered) == [(32, 40)]
+ assert verdict.padding == []
+
+
+def test_tail_gap_is_padding_only_when_alignment_forces_it() -> None:
+ padded = _info("t.Pad", parent="ffi.Object", total_size=32,
fields=(_field("v_i8", 24, 1),))
+ hole = _info("t.Hole", parent="ffi.Object", total_size=40,
fields=(_field("v_i8", 24, 1),))
+ verdicts = classify({"ffi.Object": ROOT, "t.Pad": padded, "t.Hole": hole})
+ assert verdicts["t.Pad"].is_complete
+ assert _ranges(verdicts["t.Pad"].padding) == [(25, 32)]
+ assert verdicts["t.Hole"].reason == "uncovered-bytes"
+ assert _ranges(verdicts["t.Hole"].uncovered) == [(25, 40)]
+
+
+def test_alignment_propagates_along_the_chain() -> None:
+ base = _info("t.A", parent="ffi.Object", total_size=32,
fields=(_field("a", 24, 8),))
+ wide = _info("t.B", parent="t.A", total_size=48, fields=(_field("b", 32,
16, 16),))
+ leaf = _info("t.C", parent="t.B", total_size=64, fields=(_field("c", 48,
4),))
+ verdicts = classify({"ffi.Object": ROOT, "t.A": base, "t.B": wide, "t.C":
leaf})
+ assert [verdicts[k].alignment for k in ("t.A", "t.B", "t.C")] == [8, 16,
16]
+ assert verdicts["t.C"].is_complete
+ assert _ranges(verdicts["t.C"].padding) == [(52, 64)] # the 16-byte
alignment explains the tail
+
+
+def test_no_own_metadata_is_layout_unknown_and_propagates() -> None:
+ inherited = _info("t.NoMeta", parent="ffi.Object", total_size=None)
+ child = _info("t.Child", parent="t.NoMeta", total_size=48,
fields=(_field("x", 40, 8),))
+ verdicts = classify({"ffi.Object": ROOT, "t.NoMeta": inherited, "t.Child":
child})
+ assert verdicts["t.NoMeta"].reason == "layout-unknown"
+ assert verdicts["t.NoMeta"].own_bytes is None
+ assert verdicts["t.Child"].reason == "parent-opaque"
+ assert "'t.NoMeta'" in verdicts["t.Child"].detail
+ assert "layout-unknown" in verdicts["t.Child"].detail
+ # The child's own byte facts are still recorded as evidence.
+ assert verdicts["t.Child"].fields == [FieldBytes("x", 40, 8, 8)]
+
+
+def test_field_without_byte_facts_is_layout_unknown() -> None:
+ info = _info("t.NoFacts", parent="ffi.Object", total_size=32,
fields=(_field("x", None, None),))
+ verdict = classify({"ffi.Object": ROOT, "t.NoFacts": info})["t.NoFacts"]
+ assert verdict.reason == "layout-unknown"
+ assert "'x'" in verdict.detail
+
+
+def test_field_overlap() -> None:
+ """A field placed in the parent's tail padding (Itanium ABI) cannot be
nested by value."""
+ reuse = _info("t.TailReuse", parent="t.Base", total_size=40,
fields=(_field("v_tail", 36, 4),))
+ overflow = _info("t.Overflow", parent="ffi.Object", total_size=24,
fields=(_field("a", 24, 8),))
+ verdicts = classify(
+ {"ffi.Object": ROOT, "t.Base": BASE, "t.TailReuse": reuse,
"t.Overflow": overflow}
+ )
+ assert verdicts["t.TailReuse"].reason == "field-overlap"
+ assert "'v_tail' at [36, 40) starts before byte 40" in
verdicts["t.TailReuse"].detail
+ assert verdicts["t.Overflow"].reason == "field-overlap"
+ assert "past total_size 24" in verdicts["t.Overflow"].detail
+
+
+def test_forced_opaque_only_demotes_a_complete_type() -> None:
+ inherited = _info("t.NoMeta", parent="ffi.Object", total_size=None)
+ infos = {"ffi.Object": ROOT, "t.Base": BASE, "t.NoMeta": inherited}
+ verdicts = classify(infos, forced_opaque={"t.Base", "t.NoMeta"})
+ assert verdicts["t.Base"].reason == "by-directive"
+ assert verdicts["t.Base"].own_bytes == ByteRange(HEADER, 40) # the
evidence is still there
+ assert verdicts["t.NoMeta"].reason == "layout-unknown" # a layout reason
wins over the veto
+
+
+def test_field_renderable_predicate() -> None:
+ seen: list[str] = []
+
+ def renderable(field: NamedTypeSchema) -> bool:
+ seen.append(field.name)
+ return field.name != "v_i32"
+
+ hidden = _info("t.Hidden", parent="ffi.Object", total_size=48,
fields=(_field("v_i64", 24, 8),))
+ infos = {"ffi.Object": ROOT, "t.Base": BASE, "t.Hidden": hidden}
+ verdicts = classify(infos, field_renderable=renderable)
+ assert verdicts["t.Base"].reason == "unrenderable-field"
+ assert "'v_i32'" in verdicts["t.Base"].detail
+ # A type whose bytes already fail is not asked about renderability.
+ assert verdicts["t.Hidden"].reason == "uncovered-bytes"
+ assert seen == ["v_i64", "v_i32"]
+
+
+def test_children_of_any_opaque_parent_are_parent_opaque() -> None:
+ child = _info("t.Child", parent="t.Base", total_size=48,
fields=(_field("x", 40, 8),))
+ infos = {"ffi.Object": ROOT, "t.Base": BASE, "t.Child": child}
+ verdicts = classify(infos, forced_opaque={"t.Base"})
+ assert verdicts["t.Child"].reason == "parent-opaque"
+ assert "by-directive" in verdicts["t.Child"].detail
+
+
+def test_order_does_not_matter_but_ancestors_must_be_present() -> None:
+ child = _info("t.Child", parent="t.Base", total_size=48,
fields=(_field("x", 40, 8),))
+ verdicts = classify({"t.Child": child, "t.Base": BASE, "ffi.Object": ROOT})
+ assert verdicts["t.Child"].is_complete
+ with pytest.raises(KeyError, match=r"t\.Base"):
+ classify({"t.Child": child})
+
+
+def test_coverage_report_shape(tmp_path: Path) -> None:
+ hidden = _info("t.Hidden", parent="ffi.Object", total_size=48,
fields=(_field("v_i64", 24, 8),))
+ verdicts = classify({"t.Hidden": hidden, "t.Base": BASE, "ffi.Object":
ROOT})
+ report = coverage_report(verdicts)
+ assert list(report) == ["ffi.Object", "t.Base", "t.Hidden"] # sorted
+ entry = report["t.Base"]
+ assert entry == {
+ "verdict": "complete",
+ "reason": None,
+ "detail": entry["detail"],
+ "parent": "ffi.Object",
+ "ancestors": ["ffi.Object"],
+ "total_size": 40,
+ "is_final": False,
+ "own_bytes": [24, 40],
+ "fields": [
+ {"name": "v_i64", "offset": 24, "size": 8, "alignment": 8},
+ {"name": "v_i32", "offset": 32, "size": 4, "alignment": 4},
+ ],
+ "uncovered": [],
+ "padding": [[36, 40]],
+ }
+ assert report["t.Hidden"]["reason"] == "uncovered-bytes"
+ assert report["t.Hidden"]["uncovered"] == [[32, 48]]
+
+ out = tmp_path / "coverage.json"
+ write_coverage_report(out, verdicts)
+ assert json.loads(out.read_text(encoding="utf-8")) == report
+
+
+# ---------------------------------------------------------------------------
+# The real registry: the `testing.*` fixtures exercise every path in CI.
+# ---------------------------------------------------------------------------
+
+
+def _registry_verdicts() -> dict[str, Verdict]:
+ infos = {
+ type_key: object_info_from_type_key(type_key)
+ for type_keys in collect_type_keys().values()
+ for type_key in type_keys
+ }
+ return classify(infos)
+
+
+def test_registry_complete_chain() -> None:
+ verdicts = _registry_verdicts()
+ assert verdicts["ffi.Object"].is_complete
+ base = verdicts["testing.TestCxxClassBase"]
+ derived = verdicts["testing.TestCxxClassDerived"]
+ dd = verdicts["testing.TestCxxClassDerivedDerived"]
+ assert base.is_complete and derived.is_complete and dd.is_complete
+ assert (base.own_bytes, derived.own_bytes, dd.own_bytes) == (
+ ByteRange(24, 40),
+ ByteRange(40, 56),
+ ByteRange(56, 80),
+ )
+ assert _ranges(base.padding) == [(36, 40)]
+ assert _ranges(derived.padding) == [(52, 56)]
+ assert _ranges(dd.padding) == [(73, 80)]
+ # `ffi.Function` never registered an `ObjectDef`: nothing to check against.
+ assert verdicts["ffi.Function"].reason == "layout-unknown"
+
+
+def test_registry_hidden_field() -> None:
+ """A member the registry never saw shows up as an exact byte range."""
+ verdict = _registry_verdicts()["testing.TestCxxClassHiddenField"]
+ assert verdict.reason == "uncovered-bytes"
+ assert verdict.total_size == 48
+ assert verdict.fields == [FieldBytes("v_i64", 24, 8, 8),
FieldBytes("v_i32", 40, 4, 4)]
+ assert _ranges(verdict.uncovered) == [(32, 40)]
+ assert _ranges(verdict.padding) == [(44, 48)]
+ assert verdict.is_final is True
+
+
+def test_registry_polymorphic() -> None:
+ """A vptr shifts `sizeof` by a pointer, and no polymorphism flag is needed
to see it."""
+ verdicts = _registry_verdicts()
+ verdict = verdicts["testing.TestCxxClassPolymorphic"]
+ assert verdict.reason == "uncovered-bytes"
+ assert verdict.total_size == 40 # vptr + header + int64
+ assert verdict.fields == [FieldBytes("v_i64", 24, 8, 8)]
+ assert _ranges(verdict.uncovered) == [(32, 40)]
+ # `ffi.Module` is the same shape in production code.
+ assert verdicts["ffi.Module"].reason == "uncovered-bytes"
+
+
+def test_registry_tolerates_pending_py_class_registration() -> None:
+ """A ``py_class`` whose registration never completed has no fields yet;
the sweep must not crash."""
+ parent_info = core._type_cls_to_type_info(core.Object)
+ assert parent_info is not None
+ cls = type("PendingLayout", (core.Object,), {"__slots__": ()})
+ core._register_py_class(parent_info, "testing.stub_layout.PendingLayout",
cls)
+ verdict = _registry_verdicts()["testing.stub_layout.PendingLayout"]
+ assert verdict.reason == "layout-unknown"
+ assert verdict.fields == []
+
+
+def test_cli_coverage_out_without_files(tmp_path: Path, monkeypatch:
pytest.MonkeyPatch) -> None:
+ out = tmp_path / "coverage.json"
+ monkeypatch.setattr("sys.argv", ["tvm-ffi-stubgen", "--coverage-out",
str(out)])
+ assert stub_cli.__main__() == 0
+ report = json.loads(out.read_text(encoding="utf-8"))
+ assert report["ffi.Object"]["verdict"] == "complete"
+ assert report["testing.TestCxxClassBase"]["verdict"] == "complete"
+ assert report["testing.TestCxxClassHiddenField"]["uncovered"] == [[32, 40]]
+ assert report["testing.TestCxxClassPolymorphic"]["reason"] ==
"uncovered-bytes"
+ assert report["ffi.Function"]["reason"] == "layout-unknown"
+
+
+def test_cli_still_requires_files_without_coverage(monkeypatch:
pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr("sys.argv", ["tvm-ffi-stubgen"])
+ with pytest.raises(SystemExit):
+ stub_cli.__main__()
