This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 4d6cd939e226402fffc2799df387a489b7e621da Author: Stephen Mallette <[email protected]> AuthorDate: Sun Jul 5 11:30:41 2026 -0400 tinker-review: multi-language extraction with language-scoped edges A review parsed only the dominant language, so GLV PRs that touch Java plus five language variants were mostly invisible to the graph. The extractor now parses every language a PR touches into one graph (extractMulti), and Function/Type/Test vertices carry a language property. By-name edges (calls, extends/implements, tests) resolve only within a language and external stubs are keyed by (name, language), so mixing languages never invents a cross-language edge — a Java call to next() can't land on Python's. Cross-language links stay deliberate, through the language-agnostic Step hub: every GLV's V() maps to the same Step, so "the Java V is the Python V" is a real query rather than a name-match accident. Assisted-by: Claude Code:claude-opus-4-8 --- .skills/tinker-review/references/schema.md | 17 +++++---- .../scripts/extraction/tree-sitter.js | 39 +++++++++++++++++++++ .skills/tinker-review/scripts/graph/populate.js | Bin 14177 -> 15295 bytes .skills/tinker-review/scripts/review.js | 31 ++++++++-------- .skills/tinker-review/test/extraction.test.js | 24 ++++++++++++- 5 files changed, 88 insertions(+), 23 deletions(-) diff --git a/.skills/tinker-review/references/schema.md b/.skills/tinker-review/references/schema.md index 0f787a34de..f0ffcb5ae4 100644 --- a/.skills/tinker-review/references/schema.md +++ b/.skills/tinker-review/references/schema.md @@ -15,8 +15,11 @@ PR-head worktree); `deleted: false` means it's present but an unparsed type select materialized files with `.hasNot("parsed")` and markers with `.has("parsed", false)`. -**Function** `{ name, signature, visibility, filePath, lines_start, lines_end, changed }` -A function or method. The primary unit of analysis. +**Function** `{ name, signature, visibility, filePath, language, lines_start, lines_end, changed }` +A function or method. The primary unit of analysis. `language` is the source +language (java, python, javascript, go, csharp, dart); by-name edges (`calls`, +`tests`) resolve only within a language, so a multi-language PR never invents +cross-language edges. *External stub Functions* `{ name, external: true, resolved: false, changed: false, origin?, definedIn? }` are markers created when a `calls`/`tests` edge targets a function by name that @@ -32,9 +35,10 @@ name — noise), `project` (a repo source declares a type with this name; `origin: library` calls from out-degree so ubiquitous accessor calls don't inflate hotspots. -**Type** `{ name, kind, visibility, filePath, changed }` +**Type** `{ name, kind, visibility, filePath, language, changed }` A class, interface, struct, or enum. `kind` is one of: class, interface, struct, -enum. `changed: true` if the PR modified the file it's declared in. +enum. `changed: true` if the PR modified the file it's declared in. `language` +scopes `extends`/`implements` resolution so hierarchies never cross languages. *External stub Types* `{ name, external: true, resolved: false }` are markers created when an `extends`/`implements` edge names a supertype that wasn't @@ -54,8 +58,9 @@ An ANTLR production in Gremlin.g4. ### Verification -**Test** `{ name, type, filePath }` -A test function. `type` is one of: unit, integration, suite. +**Test** `{ name, type, filePath, language }` +A test function. `type` is one of: unit, integration, suite. `language` scopes +the `tests` edge to same-language functions. **Doc** `{ path, section }` A documentation file or section that references code. diff --git a/.skills/tinker-review/scripts/extraction/tree-sitter.js b/.skills/tinker-review/scripts/extraction/tree-sitter.js index f674e63c31..065f642020 100644 --- a/.skills/tinker-review/scripts/extraction/tree-sitter.js +++ b/.skills/tinker-review/scripts/extraction/tree-sitter.js @@ -260,6 +260,7 @@ function extractFunctionsFromTree(tree, filePath, language, fileChanged) { signature: extractSignature(node, language) || name, visibility: getVisibility(node, language), filePath, + language, linesStart: node.startPosition.row + 1, linesEnd: node.endPosition.row + 1, changed: fileChanged, @@ -415,6 +416,7 @@ function extractTypesFromTree(tree, filePath, language, fileChanged) { kind: inferTypeKind(node, language), visibility: getVisibility(node, language), filePath, + language, changed: fileChanged, supertypes: extractSupertypes(node, language), }); @@ -516,6 +518,7 @@ function extractCallsFromTree(tree, filePath, language, functionRanges) { callerName: findEnclosingFunction(line, functionRanges) || "<module>", callerFile: filePath, calleeName, + language, line, }); } @@ -661,6 +664,7 @@ function parseSourceFile(parser, file, language, result) { name: fn.name, type: classifyTestType(file.path, language), filePath: file.path, + language, calledFunctions: fileCalls.filter((c) => c.callerName === fn.name).map((c) => c.calleeName), }); } @@ -841,3 +845,38 @@ export async function extract(directory, language, options = {}) { parser.delete(); return result; } + +/** + * Extract every language a PR touches into one merged result. Each language is + * parsed independently (own parser, own hierarchy-neighborhood pass) and the + * records — already stamped with `language` — are concatenated. This is what + * makes the graph multi-language; population then keeps by-name edges scoped to + * a single language so no spurious cross-language edges are invented. + * + * @param {string} directory - Absolute path to the PR worktree + * @param {string[]} languages - Languages to parse (e.g., ["java", "python"]) + * @param {object} [options] - Passed through to `extract` (e.g., changedFiles) + * @returns {Promise<ExtractionResult>} merged result; `languages` lists what was parsed + */ +export async function extractMulti(directory, languages, options = {}) { + const merged = { + languages: [], + files: [], functions: [], types: [], calls: [], imports: [], tests: [], declares: [], + hierarchyNeighborhood: { files: 0, truncated: false }, + }; + + for (const language of languages) { + if (!LANGUAGE_EXTENSIONS[language]) continue; + const one = await extract(directory, language, options); + merged.languages.push(language); + for (const key of ["files", "functions", "types", "calls", "imports", "tests", "declares"]) { + merged[key].push(...one[key]); + } + if (one.hierarchyNeighborhood) { + merged.hierarchyNeighborhood.files += one.hierarchyNeighborhood.files; + merged.hierarchyNeighborhood.truncated ||= one.hierarchyNeighborhood.truncated; + } + } + + return merged; +} diff --git a/.skills/tinker-review/scripts/graph/populate.js b/.skills/tinker-review/scripts/graph/populate.js index 8542af8d6b..75bc915eeb 100644 Binary files a/.skills/tinker-review/scripts/graph/populate.js and b/.skills/tinker-review/scripts/graph/populate.js differ diff --git a/.skills/tinker-review/scripts/review.js b/.skills/tinker-review/scripts/review.js index bca8b44193..be546d29a8 100644 --- a/.skills/tinker-review/scripts/review.js +++ b/.skills/tinker-review/scripts/review.js @@ -25,7 +25,7 @@ import { existsSync } from "node:fs"; import gremlin from "gremlin"; import { startServer, stopServer } from "./infrastructure/docker.js"; -import { extract } from "./extraction/tree-sitter.js"; +import { extractMulti } from "./extraction/tree-sitter.js"; import { populate } from "./graph/populate.js"; import { populateDiscussions } from "./graph/populate-discussions.js"; import { completeness } from "./patterns/completeness.js"; @@ -57,7 +57,10 @@ function log(msg) { process.stdout.write(`[review] ${msg}\n`); } -function detectLanguage(changedFiles) { +// Every language present among the changed files, most-changed first. The graph +// is built from all of them (extractMulti); by-name edges stay language-scoped so +// mixing languages doesn't invent cross-language edges. Falls back to ["java"]. +function detectLanguages(changedFiles) { const extCounts = new Map(); for (const file of changedFiles) { const ext = extname(file).slice(1); @@ -65,16 +68,8 @@ function detectLanguage(changedFiles) { extCounts.set(LANGUAGE_HINTS[ext], (extCounts.get(LANGUAGE_HINTS[ext]) || 0) + 1); } } - - let best = null; - let bestCount = 0; - for (const [lang, count] of extCounts) { - if (count > bestCount) { - best = lang; - bestCount = count; - } - } - return best || "java"; + const langs = [...extCounts.entries()].sort((a, b) => b[1] - a[1]).map(([lang]) => lang); + return langs.length > 0 ? langs : ["java"]; } async function getChangedFiles(repoPath, prBranch, remote = "upstream", baseBranch = "master") { @@ -193,9 +188,10 @@ export async function setup(params) { await exec("git", ["worktree", "add", worktreePath, prBranch], { cwd: repoPath }); const changedFiles = await getChangedFiles(repoPath, prBranch, remote, baseBranch); - const language = detectLanguage(changedFiles); + const languages = detectLanguages(changedFiles); + const language = languages[0]; const domains = classifyDomains(changedFiles); - log(`PR #${pr} — classified as: ${domains.join(", ")} (${language}, ${changedFiles.length} files changed)`); + log(`PR #${pr} — classified as: ${domains.join(", ")} (${languages.join("+")}, ${changedFiles.length} files changed)`); log(`Starting Gremlin Server...`); const handle = await startServer(); @@ -227,6 +223,7 @@ export async function setup(params) { worktreePath, changedFiles, language, + languages, domains, handle, connection, @@ -244,9 +241,10 @@ export async function setup(params) { export async function phase1(session) { const { pr, repoPath, remote, baseBranch = "master", prBranch, workDir, worktreePath, changedFiles, language, domains, g, a } = session; + const languages = session.languages || [language]; - log(`Phase 1: Extracting structure (${language})...`); - const extraction = await extract(worktreePath, language, { changedFiles }); + log(`Phase 1: Extracting structure (${languages.join("+")})...`); + const extraction = await extractMulti(worktreePath, languages, { changedFiles }); log(`Phase 1 complete: ${extraction.files.length} files, ${extraction.functions.length} functions, ${extraction.types.length} types`); const neighborhood = extraction.hierarchyNeighborhood; if (neighborhood && neighborhood.files > 0) { @@ -341,6 +339,7 @@ export async function phase1(session) { title: prTitle.trim(), domains, language, + languages, changedFileCount: changedFiles.length, timestamp: new Date().toISOString(), }, diff --git a/.skills/tinker-review/test/extraction.test.js b/.skills/tinker-review/test/extraction.test.js index db2c0dc3e3..dafa6bd6c9 100644 --- a/.skills/tinker-review/test/extraction.test.js +++ b/.skills/tinker-review/test/extraction.test.js @@ -28,7 +28,7 @@ import { mkdtemp, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { extract } from "../scripts/extraction/tree-sitter.js"; +import { extract, extractMulti } from "../scripts/extraction/tree-sitter.js"; async function withJavaSources(files, fn) { const dir = await mkdtemp(join(tmpdir(), "tinker-extract-")); @@ -138,6 +138,28 @@ test("changing only a subclass pulls its ancestors in as context (upward)", asyn }); }); +// Multi-language extraction (P1): every language a PR touches lands in one +// merged result, and every record is stamped with its language so population can +// keep by-name edges from crossing languages. +test("extractMulti merges every language and stamps records with language", async () => { + const dir = await mkdtemp(join(tmpdir(), "tinker-multi-")); + try { + await writeFile(join(dir, "Svc.java"), "package x;\npublic class Svc { public void foo() {} }\n"); + await writeFile(join(dir, "svc.py"), "class Svc:\n def foo(self):\n pass\n"); + const r = await extractMulti(dir, ["java", "python"], { changedFiles: ["Svc.java", "svc.py"] }); + + assert.deepEqual([...r.languages].sort(), ["java", "python"]); + const langs = new Set(r.functions.map((f) => f.language)); + assert.ok(langs.has("java") && langs.has("python"), "functions from both languages present"); + assert.equal(r.types.find((t) => t.language === "java").name, "Svc"); + assert.equal(r.types.find((t) => t.language === "python").name, "Svc"); + // Every record carries a language (no undefined leaked through the merge). + assert.ok(r.functions.every((f) => f.language) && r.types.every((t) => t.language)); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("expandHierarchy:false keeps extraction to changed files only", async () => { const dir = await mkdtemp(join(tmpdir(), "tinker-hier-")); try {
