imbajin commented on code in PR #391:
URL: https://github.com/apache/hugegraph-ai/pull/391#discussion_r4505251665


##########
ontogeny/server/ontogeny/auth/service.py:
##########
@@ -284,37 +326,33 @@ async def ensure_admin(self) -> dict[str, Any] | None:
                 return None
 
         username = os.environ.get("ONTOGENY_ADMIN_USER", "admin")
-        # the documented dev bootstrap default: a random value that is never
-        # retrievable afterwards would lock every fresh non-env deployment out
-        password = os.environ.get("ONTOGENY_ADMIN_PASSWORD") or 
BOOTSTRAP_ADMIN_PASSWORD
-        # self-heal instead of crashing: the bootstrap account may already
-        # exist as non-admin or disabled (demoted by a buggy older path), and
-        # create_user would raise "username exists" on the next restart
+        # Never silently restore privileges to a disabled or demoted account.
+        # A deployment with this inconsistent state needs explicit recovery.
         async with self.sc.sessionmaker() as session:
             row = await self._user(session, username)
             if row is not None:
-                row.is_admin = True
-                row.status = "active"
-                row.roles = sorted(set(row.roles) | {"admin"})
-                await session.commit()
-                log.warning(
-                    "re-activated the bootstrap administrator %r (it existed 
but was "
-                    "not an active admin); set ONTOGENY_ADMIN_PASSWORD and 
change it",
-                    username,
+                raise ConflictError(
+                    f"no active administrator exists and bootstrap user 
{username!r} already exists; "
+                    "restore an administrator explicitly or select a new 
ONTOGENY_ADMIN_USER"
                 )
-                return self._view(row)
+        configured_password = os.environ.get("ONTOGENY_ADMIN_PASSWORD")
+        password = configured_password or f"{secrets.randbelow(1_000_000):06d}"

Review Comment:
   Bootstrap credentials now use secrets.token_urlsafe(24), and the 
eight-character minimum is aligned across API, console, and documentation. 
Existing credentials remain valid for login. Fixed in 
84d2b5dbebf6757b240652a1e3cccaa329435a25. Combined targeted backend checks: 156 
passed, 4 skipped (PostgreSQL DSN not configured); frontend access tests: 19 
passed. The backend run emitted three aiosqlite event-loop-closed warnings.



##########
ontogeny/server/ontogeny/documents/engine.py:
##########
@@ -0,0 +1,1655 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Document ingestion orchestration: corpus -> packets -> objects -> review.
+
+The engine owns the only place where agent output becomes platform data. Two
+rules drive the whole module:
+
+* an extraction run is *replayable*: packets are persisted and re-bound without
+  another model call, so an ontology change is a rebind, not a re-bill;
+* nothing ambiguous is written silently: missing identity, conflicting values
+  and unresolved edge endpoints become review candidates.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import delete, select
+
+from ..action.models import OutboxRow, RevisionRow
+from ..core.models import DocumentCfg, ExtractCfg, ExtractedJoin
+from ..errors import NotFoundError, OOError, StoreError
+from ..stores.repo import utcnow
+from . import corpus
+from .binder import Binder, BoundObject, _jsonable
+from .models import (
+    DocumentRow,
+    ExtractAssertionRow,
+    ExtractCandidateRow,
+    ExtractOriginRow,
+    ExtractPacketRow,
+    ExtractRunRow,
+)
+from .protocol import DocInput, ExtractOptions, ExtractRequest
+from .schema import compile_extraction_schema
+
+log = logging.getLogger("ontogeny.documents")
+
+PROMPT_VERSION = "ontogeny-doc-extract/v1"
+
+
+class DocumentError(OOError):
+    code = "DOCUMENT_ERROR"
+    http_status = 422
+
+
+def _valid(value: Any) -> bool:
+    return value is not None and value != "" and value != [] and value != {}
+
+
+def _sha(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class DocumentService:
+    def __init__(self, sc) -> None:
+        self.sc = sc
+        self._locks: dict[str, Any] = {}
+        from .tasks import TaskService
+
+        self.tasks = TaskService(self)
+
+    # --------------------------------------------------------------- helpers
+
+    @property
+    def compiled(self):
+        return self.sc.compiled
+
+    def _extractor(self):
+        return getattr(self.sc, "document_extractor", None)
+
+    def document_stores(self) -> list[Any]:
+        return sorted(
+            (s for s in self.compiled.stores.values() if s.spec.type == 
"document"),
+            key=lambda s: s.metadata.name,
+        )
+
+    def store(self, name: str):
+        store = self.compiled.stores.get(name)
+        if store is None or store.spec.type != "document":
+            raise NotFoundError(f"no document store named {name!r}")
+        return store
+
+    def object_types(self, store_name: str) -> list[str]:
+        out = []
+        for name, obj in self.compiled.objects.items():
+            b = obj.spec.backing
+            if b is not None and b.store == store_name and b.sync is not None 
and b.sync.strategy == "extract":
+                out.append(name)
+        return sorted(out)
+
+    def _cfg(self, store) -> DocumentCfg:
+        return store.spec.documents or DocumentCfg()
+
+    def _root(self, store) -> Path:
+        return corpus.resolve_root(store, self.sc.package_root)
+
+    def _instructions(self, cfg: DocumentCfg) -> str | None:
+        if not cfg.prompt_file:
+            return None
+        root = self.sc.package_root
+        if not root:
+            raise DocumentError("prompt_file requires ONTOGENY_PACKAGE_ROOT")
+        path = (Path(root) / cfg.prompt_file).resolve()
+        if not str(path).startswith(str(Path(root).resolve())):
+            raise DocumentError("prompt_file must stay inside the package 
root")
+        if not path.is_file():
+            raise DocumentError(f"prompt_file not found: {cfg.prompt_file}")
+        return path.read_text(encoding="utf-8")
+
+    # ------------------------------------------------------- default source
+
+    async def ensure_default_source(self) -> dict[str, Any]:
+        """Create a generic document corpus when the package has none.
+
+        This is an explicit admin action, not a silent page-load write: adding
+        an object type changes the ontology, so the caller must ask for it.
+        """
+        existing = self.document_stores()
+        if existing:
+            return {"created": False, "store": existing[0].metadata.name}
+        root = self.sc.package_root
+        if not root:
+            raise DocumentError("ONTOGENY_PACKAGE_ROOT is not set; cannot 
create a default source")
+        package = Path(root)
+        stores_dir = package / "stores"
+        objects_dir = package / "objects"
+        stores_dir.mkdir(parents=True, exist_ok=True)
+        objects_dir.mkdir(parents=True, exist_ok=True)
+        store_name = "documents"
+        suffix = 2
+        while (stores_dir / f"{store_name}.yaml").exists() or store_name in 
self.compiled.stores:
+            store_name = f"documents-{suffix}"
+            suffix += 1
+        if (stores_dir / f"{store_name}.yaml").exists():
+            raise DocumentError(f"store resource {store_name!r} already 
exists")
+        type_name = "document-note"
+        suffix = 2
+        while (objects_dir / f"{type_name}.yaml").exists() or type_name in 
self.compiled.objects:
+            type_name = f"document-note-{suffix}"
+            suffix += 1
+        store_yaml = f"""apiVersion: ontogeny/v1
+kind: Store
+metadata:
+  name: {store_name}
+  display: Documents
+  description: Default document corpus for document intelligence
+spec:
+  type: document
+  connection: data/docs
+  documents:
+    include:
+      - "**/*.md"
+      - "**/*.txt"
+      - "**/*.pdf"
+      - "**/*.docx"
+    split: paragraph
+    language: en
+"""
+        type_yaml = f"""apiVersion: ontogeny/v1
+kind: ObjectType
+metadata:
+  name: {type_name}
+  display: Document note
+  description: A fact extracted from a document
+spec:
+  primaryKey: [note_id]
+  properties:
+    note_id:
+      type: string
+      display: Note ID
+      required: true
+      owner: document
+    title:
+      type: string
+      display: Title
+      required: true
+      owner: document
+    summary:
+      type: string
+      display: Summary
+      owner: document
+    body:
+      type: string
+      display: Body
+      owner: document
+  backing:
+    store: {store_name}
+    mode: materialized
+    source:
+      table: document-notes
+    sync:
+      strategy: extract
+    extract:
+      key: [title]
+      match: [title]
+      create: auto
+      update: fill-empty
+"""
+        (stores_dir / f"{store_name}.yaml").write_text(store_yaml, 
encoding="utf-8")
+        (objects_dir / f"{type_name}.yaml").write_text(type_yaml, 
encoding="utf-8")
+        (package / "data" / "docs").mkdir(parents=True, exist_ok=True)
+        await self.sc.publish(package)
+        return {"created": True, "store": store_name, "object_type": type_name}
+
+    # ---------------------------------------------------------------- corpus
+
+    async def _sync_documents(
+        self, session, store, cfg, scanned: list[corpus.ScannedDocument]
+    ) -> tuple[list[DocumentRow], list[str]]:
+        rows = (
+            (await session.execute(select(DocumentRow).where(DocumentRow.store 
== store.metadata.name))).scalars().all()
+        )
+        by_path = {str(r.path): r for r in rows if r.status == "active"}
+        now = utcnow()
+        active: list[DocumentRow] = []
+        changed: list[DocumentRow] = []
+        seen: set[str] = set()
+        for item in scanned:
+            key = str(item.path)
+            seen.add(key)
+            row = by_path.get(key)
+            if row is not None and row.id == item.doc_id:
+                row.last_seen_at = now
+                row.status = "active"
+                active.append(row)
+                changed.append(row)
+                continue
+            if row is not None and row.id != item.doc_id:
+                # same path, new content: the old document is removed, the new
+                # one is a different id (content-addressed)
+                row.status = "removed"
+                changed.append(row)
+            new = DocumentRow(

Review Comment:
   Previously removed content-addressed document rows are reactivated, 
including updated paths and metadata; regression tests cover re-upload and 
reverted content. Fixed in 84d2b5dbebf6757b240652a1e3cccaa329435a25. Combined 
targeted backend checks: 156 passed, 4 skipped (PostgreSQL DSN not configured); 
frontend access tests: 19 passed. The backend run emitted three aiosqlite 
event-loop-closed warnings.



##########
ontogeny/server/ontogeny/documents/engine.py:
##########
@@ -0,0 +1,1655 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Document ingestion orchestration: corpus -> packets -> objects -> review.
+
+The engine owns the only place where agent output becomes platform data. Two
+rules drive the whole module:
+
+* an extraction run is *replayable*: packets are persisted and re-bound without
+  another model call, so an ontology change is a rebind, not a re-bill;
+* nothing ambiguous is written silently: missing identity, conflicting values
+  and unresolved edge endpoints become review candidates.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import delete, select
+
+from ..action.models import OutboxRow, RevisionRow
+from ..core.models import DocumentCfg, ExtractCfg, ExtractedJoin
+from ..errors import NotFoundError, OOError, StoreError
+from ..stores.repo import utcnow
+from . import corpus
+from .binder import Binder, BoundObject, _jsonable
+from .models import (
+    DocumentRow,
+    ExtractAssertionRow,
+    ExtractCandidateRow,
+    ExtractOriginRow,
+    ExtractPacketRow,
+    ExtractRunRow,
+)
+from .protocol import DocInput, ExtractOptions, ExtractRequest
+from .schema import compile_extraction_schema
+
+log = logging.getLogger("ontogeny.documents")
+
+PROMPT_VERSION = "ontogeny-doc-extract/v1"
+
+
+class DocumentError(OOError):
+    code = "DOCUMENT_ERROR"
+    http_status = 422
+
+
+def _valid(value: Any) -> bool:
+    return value is not None and value != "" and value != [] and value != {}
+
+
+def _sha(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+class DocumentService:
+    def __init__(self, sc) -> None:
+        self.sc = sc
+        self._locks: dict[str, Any] = {}
+        from .tasks import TaskService
+
+        self.tasks = TaskService(self)
+
+    # --------------------------------------------------------------- helpers
+
+    @property
+    def compiled(self):
+        return self.sc.compiled
+
+    def _extractor(self):
+        return getattr(self.sc, "document_extractor", None)
+
+    def document_stores(self) -> list[Any]:
+        return sorted(
+            (s for s in self.compiled.stores.values() if s.spec.type == 
"document"),
+            key=lambda s: s.metadata.name,
+        )
+
+    def store(self, name: str):
+        store = self.compiled.stores.get(name)
+        if store is None or store.spec.type != "document":
+            raise NotFoundError(f"no document store named {name!r}")
+        return store
+
+    def object_types(self, store_name: str) -> list[str]:
+        out = []
+        for name, obj in self.compiled.objects.items():
+            b = obj.spec.backing
+            if b is not None and b.store == store_name and b.sync is not None 
and b.sync.strategy == "extract":
+                out.append(name)
+        return sorted(out)
+
+    def _cfg(self, store) -> DocumentCfg:
+        return store.spec.documents or DocumentCfg()
+
+    def _root(self, store) -> Path:
+        return corpus.resolve_root(store, self.sc.package_root)
+
+    def _instructions(self, cfg: DocumentCfg) -> str | None:
+        if not cfg.prompt_file:
+            return None
+        root = self.sc.package_root
+        if not root:
+            raise DocumentError("prompt_file requires ONTOGENY_PACKAGE_ROOT")
+        path = (Path(root) / cfg.prompt_file).resolve()
+        if not str(path).startswith(str(Path(root).resolve())):
+            raise DocumentError("prompt_file must stay inside the package 
root")
+        if not path.is_file():
+            raise DocumentError(f"prompt_file not found: {cfg.prompt_file}")
+        return path.read_text(encoding="utf-8")
+
+    # ------------------------------------------------------- default source
+
+    async def ensure_default_source(self) -> dict[str, Any]:
+        """Create a generic document corpus when the package has none.
+
+        This is an explicit admin action, not a silent page-load write: adding
+        an object type changes the ontology, so the caller must ask for it.
+        """
+        existing = self.document_stores()
+        if existing:
+            return {"created": False, "store": existing[0].metadata.name}
+        root = self.sc.package_root
+        if not root:
+            raise DocumentError("ONTOGENY_PACKAGE_ROOT is not set; cannot 
create a default source")
+        package = Path(root)
+        stores_dir = package / "stores"
+        objects_dir = package / "objects"
+        stores_dir.mkdir(parents=True, exist_ok=True)
+        objects_dir.mkdir(parents=True, exist_ok=True)
+        store_name = "documents"
+        suffix = 2
+        while (stores_dir / f"{store_name}.yaml").exists() or store_name in 
self.compiled.stores:
+            store_name = f"documents-{suffix}"
+            suffix += 1
+        if (stores_dir / f"{store_name}.yaml").exists():
+            raise DocumentError(f"store resource {store_name!r} already 
exists")
+        type_name = "document-note"
+        suffix = 2
+        while (objects_dir / f"{type_name}.yaml").exists() or type_name in 
self.compiled.objects:
+            type_name = f"document-note-{suffix}"
+            suffix += 1
+        store_yaml = f"""apiVersion: ontogeny/v1
+kind: Store
+metadata:
+  name: {store_name}
+  display: Documents
+  description: Default document corpus for document intelligence
+spec:
+  type: document
+  connection: data/docs
+  documents:
+    include:
+      - "**/*.md"
+      - "**/*.txt"
+      - "**/*.pdf"
+      - "**/*.docx"
+    split: paragraph
+    language: en
+"""
+        type_yaml = f"""apiVersion: ontogeny/v1
+kind: ObjectType
+metadata:
+  name: {type_name}
+  display: Document note
+  description: A fact extracted from a document
+spec:
+  primaryKey: [note_id]
+  properties:
+    note_id:
+      type: string
+      display: Note ID
+      required: true
+      owner: document
+    title:
+      type: string
+      display: Title
+      required: true
+      owner: document
+    summary:
+      type: string
+      display: Summary
+      owner: document
+    body:
+      type: string
+      display: Body
+      owner: document
+  backing:
+    store: {store_name}
+    mode: materialized
+    source:
+      table: document-notes
+    sync:
+      strategy: extract
+    extract:
+      key: [title]
+      match: [title]
+      create: auto
+      update: fill-empty
+"""
+        (stores_dir / f"{store_name}.yaml").write_text(store_yaml, 
encoding="utf-8")
+        (objects_dir / f"{type_name}.yaml").write_text(type_yaml, 
encoding="utf-8")
+        (package / "data" / "docs").mkdir(parents=True, exist_ok=True)
+        await self.sc.publish(package)
+        return {"created": True, "store": store_name, "object_type": type_name}
+
+    # ---------------------------------------------------------------- corpus
+
+    async def _sync_documents(
+        self, session, store, cfg, scanned: list[corpus.ScannedDocument]
+    ) -> tuple[list[DocumentRow], list[str]]:
+        rows = (
+            (await session.execute(select(DocumentRow).where(DocumentRow.store 
== store.metadata.name))).scalars().all()
+        )
+        by_path = {str(r.path): r for r in rows if r.status == "active"}
+        now = utcnow()
+        active: list[DocumentRow] = []
+        changed: list[DocumentRow] = []
+        seen: set[str] = set()
+        for item in scanned:
+            key = str(item.path)
+            seen.add(key)
+            row = by_path.get(key)
+            if row is not None and row.id == item.doc_id:
+                row.last_seen_at = now
+                row.status = "active"
+                active.append(row)
+                changed.append(row)
+                continue
+            if row is not None and row.id != item.doc_id:
+                # same path, new content: the old document is removed, the new
+                # one is a different id (content-addressed)
+                row.status = "removed"
+                changed.append(row)
+            new = DocumentRow(
+                id=item.doc_id,
+                store=store.metadata.name,
+                selector=None,
+                path=key,
+                media_type=item.media_type,
+                size=item.size,
+                status="active",
+                first_seen_at=now,
+                last_seen_at=now,
+            )
+            session.add(new)
+            active.append(new)
+            changed.append(new)
+        removed: list[str] = []
+        for row in rows:
+            if row.status == "active" and str(row.path) not in seen:
+                row.status = "removed"
+                removed.append(row.id)
+                changed.append(row)
+        return active, removed
+
+    # ------------------------------------------------------------- extract
+
+    def _options(self, cfg: DocumentCfg) -> ExtractOptions:
+        return ExtractOptions(
+            split=cfg.split,
+            language=cfg.language,
+            instructions=self._instructions(cfg),
+            max_chunks_per_doc=cfg.max_chunks_per_doc,
+        )
+
+    async def _packets_for(
+        self, session, doc_id: str, schema_hash: str, prompt_hash: str, model: 
str
+    ) -> list[ExtractPacketRow]:
+        rows = (
+            (
+                await session.execute(
+                    select(ExtractPacketRow)
+                    .where(
+                        ExtractPacketRow.doc_id == doc_id,
+                        ExtractPacketRow.schema_hash == schema_hash,
+                        ExtractPacketRow.prompt_hash == prompt_hash,
+                        ExtractPacketRow.model == model,
+                    )
+                    .order_by(ExtractPacketRow.chunk_index)
+                )
+            )
+            .scalars()
+            .all()
+        )
+        return list(rows)
+
+    async def _extract_doc(
+        self, session, run, doc: DocumentRow, schema, options, *, schema_hash: 
str, prompt_hash: str, model: str
+    ) -> int:
+        extractor = self._extractor()
+        if extractor is None:
+            raise DocumentError(
+                "no document extractor is configured; install/enable 
extensions/doc-extract-agent and set "
+                "ONTOGENY_DOC_AGENT_CMD or ONTOGENY_DOC_AGENT_URL"
+            )
+        path = Path(doc.path)
+        text, content_b64 = corpus.read_for_agent(path)
+        results = await extractor.extract(
+            ExtractRequest(
+                schema=schema,
+                docs=[
+                    DocInput(
+                        doc_id=doc.id,
+                        path=doc.path,
+                        text=text,
+                        name=path.name,
+                        media_type=doc.media_type,
+                        content_b64=content_b64,
+                    )
+                ],
+                options=options,
+            )
+        )
+        # replace this run's packets for the doc (a rerun of the same doc)
+        for result in results:
+            session.add(
+                ExtractPacketRow(
+                    run_id=run.id,
+                    doc_id=doc.id,
+                    doc_hash=doc.id,
+                    chunk_index=result.index,
+                    chunk_hash=_sha(result.text),
+                    schema_hash=schema_hash,
+                    prompt_hash=prompt_hash,
+                    model=model,
+                    text=result.text,
+                    graph=_jsonable(result.graph),
+                    issues=_jsonable(result.issues),
+                    status=result.status,
+                )
+            )
+        doc.chunk_count = len(results)
+        doc.extracted_at = utcnow()
+        doc.error = None
+        return len(results)
+
+    # ---------------------------------------------------------------- bind
+
+    async def _bind(self, session, run, packets: list[tuple[ExtractPacketRow, 
dict]], schema):
+        binder = Binder(self.compiled, schema, self.sc.repo, session, 
run_id=run.id)
+        return await binder.bind(packets)
+
+    # --------------------------------------------------------------- apply
+
+    async def _write_object(
+        self,
+        session,
+        bound: BoundObject,
+        *,
+        run_id: int,
+        force: bool = False,
+        principal: str,
+        created_by: str | None = None,
+        action: str = "ingest.extract",
+    ) -> str:
+        obj = self.compiled.objects[bound.object_type]
+        cfg = obj.spec.backing.extract or ExtractCfg()
+        bound.props.setdefault(obj.spec.primaryKey[0], bound.object_id)
+        current = await self.sc.repo.get(session, bound.object_type, 
bound.object_id)
+        ts = utcnow()
+        evidence = bound.evidence
+        doc_id = bound.created_by or (next(iter(bound.docs)) if bound.docs 
else "")
+        chunk = next(iter(evidence.values())).chunk_index if evidence else 0
+        # two documents disagreed about one field: the first value is kept and
+        # the disagreement is surfaced instead of letting chunk order decide
+        for prop, values in bound.alternatives.items():
+            if not values:
+                continue
+            # point the review at the document that supplied the disagreeing
+            # value, not at the one that happened to be parsed first
+            alt = bound.alternative_evidence.get(prop)
+            payload_evidence = (
+                {prop: alt.as_dict()} if alt is not None else {k: v.as_dict() 
for k, v in evidence.items()}
+            )
+            await self._candidate(
+                session,
+                run_id,
+                alt.doc_id if alt is not None else doc_id,
+                alt.chunk_index if alt is not None else chunk,
+                "object",
+                "CONFLICT",
+                {
+                    "object_type": bound.object_type,
+                    "object_id": bound.object_id,
+                    "field": prop,
+                    "current": _jsonable(bound.props.get(prop)),
+                    "proposed": _jsonable(values[0]),
+                    "alternatives": _jsonable(values),
+                    "props": _jsonable(bound.props),
+                    "evidence": payload_evidence,
+                },
+                object_type=bound.object_type,
+                object_id=bound.object_id,
+            )
+
+        if current is None:
+            if bound.missing_required:
+                await self._candidate(
+                    session,
+                    run_id,
+                    doc_id,
+                    chunk,
+                    "object",
+                    "MISSING_REQUIRED",
+                    {
+                        "object_type": bound.object_type,
+                        "object_id": bound.object_id,
+                        "props": _jsonable(bound.props),
+                        "missing": bound.missing_required,
+                        "evidence": {k: v.as_dict() for k, v in 
evidence.items()},
+                    },
+                    object_type=bound.object_type,
+                    object_id=bound.object_id,
+                )
+                return "candidate"
+            if not force and cfg.create != "auto":
+                await self._candidate(
+                    session,
+                    run_id,
+                    doc_id,
+                    chunk,
+                    "object",
+                    "NEW_OBJECT",
+                    {
+                        "object_type": bound.object_type,
+                        "object_id": bound.object_id,
+                        "props": _jsonable(bound.props),
+                        "evidence": {k: v.as_dict() for k, v in 
evidence.items()},
+                    },
+                    object_type=bound.object_type,
+                    object_id=bound.object_id,
+                )
+                return "candidate"
+            outcome = await self.sc.repo.apply_extracted(session, 
bound.object_type, bound.props, ts)
+            created = True
+        else:
+            values: dict[str, Any] = {}
+            for prop, value in bound.props.items():
+                if prop in 
self.compiled.objects[bound.object_type].spec.primaryKey:
+                    continue
+                if not _valid(value):
+                    continue
+                owner = self.compiled.owner(bound.object_type, prop)
+                # even a human acceptance cannot write a system-of-record
+                # field: sync owns it, and the review gate is not a second
+                # sync with better intentions
+                if owner == "source" and not cfg.include_source:
+                    continue
+                existing = current.get(self.compiled.col(prop))
+                if force:
+                    values[prop] = value
+                    continue
+                if cfg.update == "never":
+                    continue
+                if cfg.update == "fill-empty" and _valid(existing):
+                    if existing != value:
+                        await self._candidate(
+                            session,
+                            run_id,
+                            doc_id,
+                            chunk,
+                            "object",
+                            "CONFLICT",
+                            {
+                                "object_type": bound.object_type,
+                                "object_id": bound.object_id,
+                                "field": prop,
+                                "current": _jsonable(existing),
+                                "proposed": _jsonable(value),
+                                "props": _jsonable(bound.props),
+                                "evidence": {k: v.as_dict() for k, v in 
evidence.items()},
+                            },
+                            object_type=bound.object_type,
+                            object_id=bound.object_id,
+                        )
+                    continue
+                values[prop] = value
+            if not values:
+                return "noop"
+            values[obj.spec.primaryKey[0]] = bound.object_id
+            outcome = await self.sc.repo.apply_extracted(session, 
bound.object_type, values, ts)
+            created = False
+
+        if created and outcome == "inserted":
+            # durable origin: assertions may later be deleted with their
+            # document, but "the corpus created this row" must survive
+            origin = await session.get(ExtractOriginRow, (bound.object_type, 
bound.object_id))
+            if origin is None:
+                session.add(
+                    ExtractOriginRow(
+                        object_type=bound.object_type,
+                        object_id=bound.object_id,
+                        doc_id=doc_id,
+                        run_id=run_id,
+                    )
+                )
+        if outcome in ("inserted", "updated"):
+            payload_props = bound.props if created else values
+            session.add(
+                OutboxRow(
+                    object_type=bound.object_type,
+                    object_id=bound.object_id,
+                    op="upsert",
+                    payload={"props": _jsonable(payload_props), "origin": 
"document", "run_id": run_id},
+                )
+            )
+            session.add(
+                RevisionRow(
+                    action=action,
+                    object_type=bound.object_type,
+                    object_id=bound.object_id,
+                    principal=principal,
+                    params={"run_id": run_id, "doc_id": doc_id},
+                    before=_jsonable(current) if current else None,
+                    after=_jsonable(bound.props),
+                    outcome="executed",
+                    message=f"extracted from {doc_id[:12]}" if doc_id else 
"extracted",
+                )
+            )
+            written = bound.props if created else values
+            for prop, value in written.items():
+                docs = bound.sources.get(prop) or bound.docs or {doc_id}
+                for source_doc in docs:
+                    ev = evidence.get(prop)
+                    await session.execute(
+                        delete(ExtractAssertionRow).where(
+                            ExtractAssertionRow.doc_id == source_doc,
+                            ExtractAssertionRow.kind == "object",
+                            ExtractAssertionRow.object_type == 
bound.object_type,
+                            ExtractAssertionRow.object_id == bound.object_id,
+                            ExtractAssertionRow.prop == prop,
+                        )
+                    )
+                    session.add(
+                        ExtractAssertionRow(
+                            run_id=run_id,
+                            doc_id=source_doc,
+                            chunk_index=ev.chunk_index if ev else 0,
+                            kind="object",
+                            object_type=bound.object_type,
+                            object_id=bound.object_id,
+                            prop=prop,
+                            value=_jsonable(value),
+                            evidence=ev.as_dict() if ev else {},
+                            created_object=bool(created and (created_by in 
(None, source_doc))),
+                        )
+                    )
+        return outcome
+
+    async def _candidate(
+        self,
+        session,
+        run_id: int,
+        doc_id: str,
+        chunk: int,
+        kind: str,
+        reason: str,
+        payload: dict[str, Any],
+        *,
+        object_type: str | None = None,
+        link: str | None = None,
+        object_id: str | None = None,
+    ) -> None:
+        material = "|".join(
+            str(x)
+            for x in (
+                kind,
+                reason,
+                object_type,
+                link,
+                object_id,
+                doc_id,
+                payload.get("field"),
+                payload.get("proposed"),
+                payload.get("src_id"),
+                payload.get("dst_id"),
+            )
+        )
+        signature = _sha(material)
+        existing = (
+            (
+                await session.execute(
+                    select(ExtractCandidateRow)
+                    .where(
+                        ExtractCandidateRow.signature == signature,
+                        ExtractCandidateRow.status == "pending",

Review Comment:
   Rejected candidate signatures are retained and no longer recreated by a 
subsequent ingestion; a regression test covers repeated ingestion. Fixed in 
84d2b5dbebf6757b240652a1e3cccaa329435a25. Combined targeted backend checks: 156 
passed, 4 skipped (PostgreSQL DSN not configured); frontend access tests: 19 
passed. The backend run emitted three aiosqlite event-loop-closed warnings.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to