Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package tree-sitter for openSUSE:Factory checked in at 2025-04-15 16:44:49 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/tree-sitter (Old) and /work/SRC/openSUSE:Factory/.tree-sitter.new.1907 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "tree-sitter" Tue Apr 15 16:44:49 2025 rev:21 rq:1268905 version:0.25.3 Changes: -------- --- /work/SRC/openSUSE:Factory/tree-sitter/tree-sitter.changes 2025-02-05 12:39:58.203421508 +0100 +++ /work/SRC/openSUSE:Factory/.tree-sitter.new.1907/tree-sitter.changes 2025-04-15 16:47:48.867898829 +0200 @@ -1,0 +2,21 @@ +Sat Apr 12 10:32:50 UTC 2025 - Michal Vyskocil <michal.vysko...@gmail.com> + +- update to 0.25.3: + Full changelog: https://github.com/tree-sitter/tree-sitter/releases/tag/v0.25.3 + Bug Fixes + * Fixed an infinite loop that could happen in some grammars during error + recovery if the end of the file was reached. + * Fixed a parser-generation error where internal character set constants were + given external linkage, which could cause linking errors on some platforms + if multiple Tree-sitter parsers were linked into the same application. +- update to 0.25.2: + Full changelog: https://github.com/tree-sitter/tree-sitter/releases/tag/v0.25.2 + Bug Fixes + * Fixed a crash that could occur when loading WASM-compiled languages that + were generated with an earlier version of the Tree-sitter CLI. + * Fixed incorrect tokenization when using WASM-compiled languages, if the + language's external scanner did not assign to result_symbol + * Fixed an infinite loop that could occur if external scanners returned + empty extra tokens + +------------------------------------------------------------------- Old: ---- tree-sitter-0.25.1.tar.gz New: ---- tree-sitter-0.25.3.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ tree-sitter.spec ++++++ --- /var/tmp/diff_new_pack.ZeV7bG/_old 2025-04-15 16:47:49.835939363 +0200 +++ /var/tmp/diff_new_pack.ZeV7bG/_new 2025-04-15 16:47:49.835939363 +0200 @@ -18,7 +18,7 @@ %define somajor 0_25 Name: tree-sitter -Version: 0.25.1 +Version: 0.25.3 Release: 0 Summary: An incremental parsing system for programming tools License: GPL-2.0-only AND MIT ++++++ tree-sitter-0.25.1.tar.gz -> tree-sitter-0.25.3.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/.github/workflows/ci.yml new/tree-sitter-0.25.3/.github/workflows/ci.yml --- old/tree-sitter-0.25.1/.github/workflows/ci.yml 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/.github/workflows/ci.yml 2025-03-05 01:03:16.000000000 +0100 @@ -32,6 +32,7 @@ uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: stable + components: clippy, rustfmt - name: Set up nightly Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 @@ -44,6 +45,11 @@ make lint make lint-web + - name: Lint files (Rust nightly) + continue-on-error: true + run: | + make lint-nightly + sanitize: uses: ./.github/workflows/sanitize.yml diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/Cargo.lock new/tree-sitter-0.25.3/Cargo.lock --- old/tree-sitter-0.25.1/Cargo.lock 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/Cargo.lock 2025-03-05 01:03:16.000000000 +0100 @@ -2044,7 +2044,7 @@ [[package]] name = "tree-sitter" -version = "0.25.1" +version = "0.25.3" dependencies = [ "bindgen", "cc", @@ -2058,7 +2058,7 @@ [[package]] name = "tree-sitter-cli" -version = "0.25.1" +version = "0.25.3" dependencies = [ "ansi_colours", "anstyle", @@ -2110,7 +2110,7 @@ [[package]] name = "tree-sitter-config" -version = "0.25.1" +version = "0.25.3" dependencies = [ "anyhow", "etcetera", @@ -2120,7 +2120,7 @@ [[package]] name = "tree-sitter-generate" -version = "0.25.1" +version = "0.25.3" dependencies = [ "anyhow", "heck", @@ -2141,7 +2141,7 @@ [[package]] name = "tree-sitter-highlight" -version = "0.25.1" +version = "0.25.3" dependencies = [ "regex", "streaming-iterator", @@ -2151,11 +2151,11 @@ [[package]] name = "tree-sitter-language" -version = "0.1.4" +version = "0.1.5" [[package]] name = "tree-sitter-loader" -version = "0.25.1" +version = "0.25.3" dependencies = [ "anyhow", "cc", @@ -2178,7 +2178,7 @@ [[package]] name = "tree-sitter-tags" -version = "0.25.1" +version = "0.25.3" dependencies = [ "memchr", "regex", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/Cargo.toml new/tree-sitter-0.25.3/Cargo.toml --- old/tree-sitter-0.25.1/Cargo.toml 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/Cargo.toml 2025-03-05 01:03:16.000000000 +0100 @@ -13,7 +13,7 @@ resolver = "2" [workspace.package] -version = "0.25.1" +version = "0.25.3" authors = [ "Max Brunsfeld <maxbrunsf...@gmail.com>", "Amaan Qureshi <amaan...@gmail.com>", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/Makefile new/tree-sitter-0.25.3/Makefile --- old/tree-sitter-0.25.1/Makefile 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/Makefile 2025-03-05 01:03:16.000000000 +0100 @@ -2,7 +2,7 @@ $(error Windows is not supported) endif -VERSION := 0.25.1 +VERSION := 0.25.3 DESCRIPTION := An incremental parsing system for programming tools HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/ @@ -107,6 +107,12 @@ cargo update --workspace --locked --quiet cargo check --workspace --all-targets cargo +nightly fmt --all --check + cargo +stable clippy --workspace --all-targets -- -D warnings + +lint-nightly: + cargo update --workspace --locked --quiet + cargo check --workspace --all-targets + cargo +nightly fmt --all --check cargo +nightly clippy --workspace --all-targets -- -D warnings lint-web: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/build.zig.zon new/tree-sitter-0.25.3/build.zig.zon --- old/tree-sitter-0.25.1/build.zig.zon 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/build.zig.zon 2025-03-05 01:03:16.000000000 +0100 @@ -1,6 +1,6 @@ .{ .name = "tree-sitter", - .version = "0.25.1", + .version = "0.25.3", .paths = .{ "build.zig", "build.zig.zon", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/generate/src/render.rs new/tree-sitter-0.25.3/cli/generate/src/render.rs --- old/tree-sitter-0.25.1/cli/generate/src/render.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/generate/src/render.rs 2025-03-05 01:03:16.000000000 +0100 @@ -1111,7 +1111,11 @@ return; } - add_line!(self, "const TSCharacterRange {}[] = {{", info.constant_name); + add_line!( + self, + "static const TSCharacterRange {}[] = {{", + info.constant_name + ); indent!(self); for (ix, range) in characters.ranges().enumerate() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/generate/src/rules.rs new/tree-sitter-0.25.3/cli/generate/src/rules.rs --- old/tree-sitter-0.25.1/cli/generate/src/rules.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/generate/src/rules.rs 2025-03-05 01:03:16.000000000 +0100 @@ -306,7 +306,6 @@ } impl From<Symbol> for Rule { - #[must_use] fn from(symbol: Symbol) -> Self { Self::Symbol(symbol) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/npm/package.json new/tree-sitter-0.25.3/cli/npm/package.json --- old/tree-sitter-0.25.1/cli/npm/package.json 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/npm/package.json 2025-03-05 01:03:16.000000000 +0100 @@ -1,6 +1,6 @@ { "name": "tree-sitter-cli", - "version": "0.25.1", + "version": "0.25.3", "author": { "name": "Max Brunsfeld", "email": "maxbrunsf...@gmail.com" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/src/fuzz/mod.rs new/tree-sitter-0.25.3/cli/src/fuzz/mod.rs --- old/tree-sitter-0.25.1/cli/src/fuzz/mod.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/src/fuzz/mod.rs 2025-03-05 01:03:16.000000000 +0100 @@ -56,7 +56,9 @@ pub fn new_seed() -> usize { int_env_var("TREE_SITTER_SEED").unwrap_or_else(|| { let mut rng = rand::thread_rng(); - rng.gen::<usize>() + let seed = rng.gen::<usize>(); + eprintln!("Seed: {seed}"); + seed }) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/src/tests/async_context_test.rs new/tree-sitter-0.25.3/cli/src/tests/async_context_test.rs --- old/tree-sitter-0.25.1/cli/src/tests/async_context_test.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/src/tests/async_context_test.rs 2025-03-05 01:03:16.000000000 +0100 @@ -238,7 +238,7 @@ SimpleYieldNow { yielded: false }.await; } -pub fn noop_waker() -> Waker { +pub const fn noop_waker() -> Waker { const VTABLE: RawWakerVTable = RawWakerVTable::new( // Cloning just returns a new no-op raw waker |_| RAW, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/src/tests/corpus_test.rs new/tree-sitter-0.25.3/cli/src/tests/corpus_test.rs --- old/tree-sitter-0.25.1/cli/src/tests/corpus_test.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/src/tests/corpus_test.rs 2025-03-05 01:03:16.000000000 +0100 @@ -23,7 +23,7 @@ }; #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_bash(seed: usize) { +fn test_corpus_for_bash_language(seed: usize) { test_language_corpus( "bash", seed, @@ -39,73 +39,77 @@ } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_c(seed: usize) { +fn test_corpus_for_c_language(seed: usize) { test_language_corpus("c", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_cpp(seed: usize) { +fn test_corpus_for_cpp_language(seed: usize) { test_language_corpus("cpp", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_embedded_template(seed: usize) { +fn test_corpus_for_embedded_template_language(seed: usize) { test_language_corpus("embedded-template", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_go(seed: usize) { +fn test_corpus_for_go_language(seed: usize) { test_language_corpus("go", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_html(seed: usize) { +fn test_corpus_for_html_language(seed: usize) { test_language_corpus("html", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_java(seed: usize) { - test_language_corpus("java", seed, None, None); +fn test_corpus_for_java_language(seed: usize) { + test_language_corpus( + "java", + seed, + Some(&["java - corpus - expressions - switch with unnamed pattern variable"]), + None, + ); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_javascript(seed: usize) { +fn test_corpus_for_javascript_language(seed: usize) { test_language_corpus("javascript", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_json(seed: usize) { +fn test_corpus_for_json_language(seed: usize) { test_language_corpus("json", seed, None, None); } -#[ignore] #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_php(seed: usize) { - test_language_corpus("php", seed, None, None); +fn test_corpus_for_php_language(seed: usize) { + test_language_corpus("php", seed, None, Some("php")); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_python(seed: usize) { +fn test_corpus_for_python_language(seed: usize) { test_language_corpus("python", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_ruby(seed: usize) { +fn test_corpus_for_ruby_language(seed: usize) { test_language_corpus("ruby", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_rust(seed: usize) { +fn test_corpus_for_rust_language(seed: usize) { test_language_corpus("rust", seed, None, None); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_typescript(seed: usize) { +fn test_corpus_for_typescript_language(seed: usize) { test_language_corpus("typescript", seed, None, Some("typescript")); } #[test_with_seed(retry=10, seed=*START_SEED, seed_fn=new_seed)] -fn test_corpus_for_tsx(seed: usize) { +fn test_corpus_for_tsx_language(seed: usize) { test_language_corpus("typescript", seed, None, Some("tsx")); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/cli/src/tests/language_test.rs new/tree-sitter-0.25.3/cli/src/tests/language_test.rs --- old/tree-sitter-0.25.1/cli/src/tests/language_test.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/cli/src/tests/language_test.rs 2025-03-05 01:03:16.000000000 +0100 @@ -152,6 +152,7 @@ "_literal_pattern", "captured_pattern", "const_block", + "generic_pattern", "identifier", "macro_invocation", "mut_pattern", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/CMakeLists.txt new/tree-sitter-0.25.3/lib/CMakeLists.txt --- old/tree-sitter-0.25.1/lib/CMakeLists.txt 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/CMakeLists.txt 2025-03-05 01:03:16.000000000 +0100 @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.13) project(tree-sitter - VERSION "0.25.1" + VERSION "0.25.3" DESCRIPTION "An incremental parsing system for programming tools" HOMEPAGE_URL "https://tree-sitter.github.io/tree-sitter/"; LANGUAGES C) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/binding_web/package-lock.json new/tree-sitter-0.25.3/lib/binding_web/package-lock.json --- old/tree-sitter-0.25.1/lib/binding_web/package-lock.json 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/binding_web/package-lock.json 2025-03-05 01:03:16.000000000 +0100 @@ -1,12 +1,12 @@ { "name": "web-tree-sitter", - "version": "0.25.0", + "version": "0.25.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "web-tree-sitter", - "version": "0.25.0", + "version": "0.25.1", "license": "MIT", "devDependencies": { "@eslint/js": "^9.19.0", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/binding_web/package.json new/tree-sitter-0.25.3/lib/binding_web/package.json --- old/tree-sitter-0.25.1/lib/binding_web/package.json 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/binding_web/package.json 2025-03-05 01:03:16.000000000 +0100 @@ -1,6 +1,6 @@ { "name": "web-tree-sitter", - "version": "0.25.1", + "version": "0.25.3", "description": "Tree-sitter bindings for the web", "repository": "https://github.com/tree-sitter/tree-sitter";, "homepage": "https://github.com/tree-sitter/tree-sitter/tree/master/lib/binding_web";, @@ -19,11 +19,13 @@ "exports": { ".": { "import": "./tree-sitter.js", - "require": "./tree-sitter.cjs" + "require": "./tree-sitter.cjs", + "types": "./web-tree-sitter.d.ts" }, "./debug": { "import": "./debug/tree-sitter.js", - "require": "./debug/tree-sitter.cjs" + "require": "./debug/tree-sitter.cjs", + "types": "./web-tree-sitter.d.ts" } }, "types": "web-tree-sitter.d.ts", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/binding_web/src/node.ts new/tree-sitter-0.25.3/lib/binding_web/src/node.ts --- old/tree-sitter-0.25.1/lib/binding_web/src/node.ts 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/binding_web/src/node.ts 2025-03-05 01:03:16.000000000 +0100 @@ -416,6 +416,11 @@ // Convert the type strings to numeric type symbols const symbols: number[] = []; const typesBySymbol = this.tree.language.types; + for (const node_type of types) { + if (node_type == "ERROR") { + symbols.push(65535); // Internally, ts_builtin_sym_error is -1, which is UINT_16MAX + } + } for (let i = 0, n = typesBySymbol.length; i < n; i++) { if (types.includes(typesBySymbol[i])) { symbols.push(i); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/binding_web/test/language.test.ts new/tree-sitter-0.25.3/lib/binding_web/test/language.test.ts --- old/tree-sitter-0.25.1/lib/binding_web/test/language.test.ts 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/binding_web/test/language.test.ts 2025-03-05 01:03:16.000000000 +0100 @@ -89,6 +89,7 @@ '_literal_pattern', 'captured_pattern', 'const_block', + 'generic_pattern', 'identifier', 'macro_invocation', 'mut_pattern', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/binding_web/test/node.test.ts new/tree-sitter-0.25.3/lib/binding_web/test/node.test.ts --- old/tree-sitter-0.25.1/lib/binding_web/test/node.test.ts 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/binding_web/test/node.test.ts 2025-03-05 01:03:16.000000000 +0100 @@ -63,7 +63,7 @@ tree = parser.parse('x10 + 1000')!; expect(tree.rootNode.children).toHaveLength(1); const sumNode = tree.rootNode.firstChild!.firstChild!; - expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number' ]); + expect(sumNode.children.map(child => child!.type)).toEqual(['identifier', '+', 'number']); }); }); @@ -449,6 +449,24 @@ }); }); + describe('.descendantsOfType("ERROR")', () => { + it('finds all of the descendants of an ERROR node', () => { + tree = parser.parse( + `if ({a: 'b'} {c: 'd'}) { + // ^ ERROR + x = function(a) { b; } function(c) { d; } + }` + )!; + const errorNode = tree.rootNode; + const descendants = errorNode.descendantsOfType('ERROR'); + expect( + descendants.map((node) => node!.startIndex) + ).toEqual( + [4] + ); + }); + }); + describe('.descendantsOfType', () => { it('finds all descendants of a given type in the given range', () => { tree = parser.parse('a + 1 * b * 2 + c + 3')!; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/language/Cargo.toml new/tree-sitter-0.25.3/lib/language/Cargo.toml --- old/tree-sitter-0.25.1/lib/language/Cargo.toml 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/language/Cargo.toml 2025-03-05 01:03:16.000000000 +0100 @@ -1,10 +1,10 @@ [package] name = "tree-sitter-language" description = "The tree-sitter Language type, used by the library and by language implementations" -version = "0.1.4" +version = "0.1.5" authors.workspace = true edition.workspace = true -rust-version.workspace = true +rust-version = "1.76" readme = "README.md" homepage.workspace = true repository.workspace = true diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/get_changed_ranges.c new/tree-sitter-0.25.3/lib/src/get_changed_ranges.c --- old/tree-sitter-0.25.1/lib/src/get_changed_ranges.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/get_changed_ranges.c 2025-03-05 01:03:16.000000000 +0100 @@ -108,6 +108,7 @@ const TSLanguage *language; unsigned visible_depth; bool in_padding; + Subtree prev_external_token; } Iterator; static Iterator iterator_new( @@ -127,6 +128,7 @@ .language = language, .visible_depth = 1, .in_padding = false, + .prev_external_token = NULL_SUBTREE, }; } @@ -244,6 +246,10 @@ position = child_right; if (!ts_subtree_extra(*child)) structural_child_index++; + Subtree last_external_token = ts_subtree_last_external_token(*child); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } } } while (did_descend); @@ -268,6 +274,10 @@ const Subtree *parent = array_back(&self->cursor.stack)->subtree; uint32_t child_index = entry.child_index + 1; + Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree); + if (last_external_token.ptr) { + self->prev_external_token = last_external_token; + } if (ts_subtree_child_count(*parent) > child_index) { Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree)); uint32_t structural_child_index = entry.structural_child_index; @@ -313,29 +323,41 @@ TSSymbol new_alias_symbol = 0; iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start); iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start); + TSSymbol old_symbol = ts_subtree_symbol(old_tree); + TSSymbol new_symbol = ts_subtree_symbol(new_tree); if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches; if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers; + if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers; + + uint32_t old_size = ts_subtree_size(old_tree).bytes; + uint32_t new_size = ts_subtree_size(new_tree).bytes; + TSStateId old_state = ts_subtree_parse_state(old_tree); + TSStateId new_state = ts_subtree_parse_state(new_tree); + bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree); + bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree); + uint32_t old_error_cost = ts_subtree_error_cost(old_tree); + uint32_t new_error_cost = ts_subtree_error_cost(new_tree); if ( - old_alias_symbol == new_alias_symbol && - ts_subtree_symbol(old_tree) == ts_subtree_symbol(new_tree) + old_start != new_start || + old_symbol == ts_builtin_sym_error || + old_size != new_size || + old_state == TS_TREE_STATE_NONE || + new_state == TS_TREE_STATE_NONE || + ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) || + old_error_cost != new_error_cost || + old_has_external_tokens != new_has_external_tokens || + ts_subtree_has_changes(old_tree) || + ( + old_has_external_tokens && + !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token) + ) ) { - if (old_start == new_start && - !ts_subtree_has_changes(old_tree) && - ts_subtree_symbol(old_tree) != ts_builtin_sym_error && - ts_subtree_size(old_tree).bytes == ts_subtree_size(new_tree).bytes && - ts_subtree_parse_state(old_tree) != TS_TREE_STATE_NONE && - ts_subtree_parse_state(new_tree) != TS_TREE_STATE_NONE && - (ts_subtree_parse_state(old_tree) == ERROR_STATE) == - (ts_subtree_parse_state(new_tree) == ERROR_STATE)) { - return IteratorMatches; - } else { - return IteratorMayDiffer; - } + return IteratorMayDiffer; } - return IteratorDiffers; + return IteratorMatches; } #ifdef DEBUG_GET_CHANGED_RANGES @@ -348,8 +370,8 @@ "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])", name, self->in_padding ? "(p)" : " ", self->visible_depth, - start.row + 1, start.column, - end.row + 1, end.column + start.row, start.column, + end.row, end.column ); } #endif @@ -380,7 +402,7 @@ do { #ifdef DEBUG_GET_CHANGED_RANGES - printf("At [%-2u, %-2u] Compare ", position.extent.row + 1, position.extent.column); + printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column); iterator_print_state(&old_iter); printf("\tvs\t"); iterator_print_state(&new_iter); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/parser.c new/tree-sitter-0.25.3/lib/src/parser.c --- old/tree-sitter-0.25.1/lib/src/parser.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/parser.c 2025-03-05 01:03:16.000000000 +0100 @@ -556,27 +556,29 @@ external_scanner_state_len ); - // When recovering from an error, ignore any zero-length external tokens - // unless they have changed the external scanner's state. This helps to - // avoid infinite loops which could otherwise occur, because the lexer is - // looking for any possible token, instead of looking for the specific set of - // tokens that are valid in some parse state. + // Avoid infinite loops caused by the external scanner returning empty tokens. + // Empty tokens are needed in some circumstances, e.g. indent/dedent tokens + // in Python. Ignore the following classes of empty tokens: // - // Note that it's possible that the token end position may be *before* the - // original position of the lexer because of the way that tokens are positioned - // at included range boundaries: when a token is terminated at the start of - // an included range, it is marked as ending at the *end* of the preceding - // included range. + // * Tokens produced during error recovery. When recovering from an error, + // all tokens are allowed, so it's easy to accidentally return unwanted + // empty tokens. + // * Tokens that are marked as 'extra' in the grammar. These don't change + // the parse state, so they would definitely cause an infinite loop. if ( self->lexer.token_end_position.bytes <= current_position.bytes && - (error_mode || !ts_stack_has_advanced_since_error(self->stack, version)) && !external_scanner_state_changed ) { - LOG( - "ignore_empty_external_token symbol:%s", - SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]) - ) - found_token = false; + TSSymbol symbol = self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]; + TSStateId next_parse_state = ts_language_next_state(self->language, parse_state, symbol); + bool token_is_extra = (next_parse_state == parse_state); + if (error_mode || !ts_stack_has_advanced_since_error(self->stack, version) || token_is_extra) { + LOG( + "ignore_empty_external_token symbol:%s", + SYM_NAME(self->language->external_scanner.symbol_map[self->lexer.data.result_symbol]) + ); + found_token = false; + } } } @@ -947,6 +949,7 @@ // children. StackSliceArray pop = ts_stack_pop_count(self->stack, version, count); uint32_t removed_version_count = 0; + uint32_t halted_version_count = ts_stack_halted_version_count(self->stack); for (uint32_t i = 0; i < pop.size; i++) { StackSlice slice = pop.contents[i]; StackVersion slice_version = slice.version - removed_version_count; @@ -955,11 +958,12 @@ // will all be sorted and truncated at the end of the outer parsing loop. // Allow the maximum version count to be temporarily exceeded, but only // by a limited threshold. - if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW) { + if (slice_version > MAX_VERSION_COUNT + MAX_VERSION_COUNT_OVERFLOW + halted_version_count) { ts_stack_remove_version(self->stack, slice_version); ts_subtree_array_delete(&self->tree_pool, &slice.subtrees); removed_version_count++; while (i + 1 < pop.size) { + LOG("aborting reduce with too many versions") StackSlice next_slice = pop.contents[i + 1]; if (next_slice.version != slice.version) break; ts_subtree_array_delete(&self->tree_pool, &next_slice.subtrees); @@ -1316,10 +1320,23 @@ // and subsequently halted. Remove those versions. for (unsigned i = previous_version_count; i < ts_stack_version_count(self->stack); i++) { if (!ts_stack_is_active(self->stack, i)) { + LOG("removed paused version:%u", i); ts_stack_remove_version(self->stack, i--); + LOG_STACK(); } } + // If the parser is still in the error state at the end of the file, just wrap everything + // in an ERROR node and terminate. + if (ts_subtree_is_eof(lookahead)) { + LOG("recover_eof"); + SubtreeArray children = array_new(); + Subtree parent = ts_subtree_new_error_node(&children, false, self->language); + ts_stack_push(self->stack, version, parent, false, 1); + ts_parser__accept(self, version, lookahead); + return; + } + // If strategy 1 succeeded, a new stack version will have been created which is able to handle // the current lookahead token. Now, in addition, try strategy 2 described above: skip the // current lookahead token by wrapping it in an ERROR node. @@ -1340,17 +1357,6 @@ return; } - // If the parser is still in the error state at the end of the file, just wrap everything - // in an ERROR node and terminate. - if (ts_subtree_is_eof(lookahead)) { - LOG("recover_eof"); - SubtreeArray children = array_new(); - Subtree parent = ts_subtree_new_error_node(&children, false, self->language); - ts_stack_push(self->stack, version, parent, false, 1); - ts_parser__accept(self, version, lookahead); - return; - } - // Do not recover if the result would clearly be worse than some existing stack version. unsigned new_cost = current_error_cost + ERROR_COST_PER_SKIPPED_TREE + @@ -1616,6 +1622,7 @@ // an ambiguous state. REDUCE actions always create a new stack // version, whereas SHIFT actions update the existing stack version // and terminate this loop. + bool did_reduce = false; StackVersion last_reduction_version = STACK_VERSION_NONE; for (uint32_t i = 0; i < table_entry.action_count; i++) { TSParseAction action = table_entry.actions[i]; @@ -1651,6 +1658,7 @@ action.reduce.dynamic_precedence, action.reduce.production_id, is_fragile, end_of_non_terminal_extra ); + did_reduce = true; if (reduction_version != STACK_VERSION_NONE) { last_reduction_version = reduction_version; } @@ -1702,9 +1710,12 @@ continue; } - // A non-terminal extra rule was reduced and merged into an existing - // stack version. This version can be discarded. - if (!lookahead.ptr) { + // A reduction was performed, but was merged into an existing stack version. + // This version can be discarded. + if (did_reduce) { + if (lookahead.ptr) { + ts_subtree_release(&self->tree_pool, lookahead); + } ts_stack_halt(self->stack, version); return true; } @@ -1753,7 +1764,7 @@ // versions that exist. If some other version advances successfully, then // this version can simply be removed. But if all versions end up paused, // then error recovery is needed. - LOG("detect_error"); + LOG("detect_error lookahead:%s", TREE_NAME(lookahead)); ts_stack_pause(self->stack, version, lookahead); return true; } @@ -1842,6 +1853,7 @@ has_unpaused_version = true; } else { ts_stack_remove_version(self->stack, i); + made_changes = true; i--; n--; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/portable/endian.h new/tree-sitter-0.25.3/lib/src/portable/endian.h --- old/tree-sitter-0.25.1/lib/src/portable/endian.h 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/portable/endian.h 2025-03-05 01:03:16.000000000 +0100 @@ -18,16 +18,20 @@ #if defined(HAVE_ENDIAN_H) || \ defined(__linux__) || \ defined(__GNU__) || \ + defined(__NetBSD__) || \ defined(__OpenBSD__) || \ defined(__CYGWIN__) || \ defined(__MSYS__) || \ defined(__EMSCRIPTEN__) +#if defined(__NetBSD__) +#define _NETBSD_SOURCE 1 +#endif + # include <endian.h> #elif defined(HAVE_SYS_ENDIAN_H) || \ defined(__FreeBSD__) || \ - defined(__NetBSD__) || \ defined(__DragonFly__) # include <sys/endian.h> diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/query.c new/tree-sitter-0.25.3/lib/src/query.c --- old/tree-sitter-0.25.1/lib/src/query.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/query.c 2025-03-05 01:03:16.000000000 +0100 @@ -1,3 +1,12 @@ +/* + * On NetBSD, defining standard requirements like this removes symbols + * from the namespace; however, we need non-standard symbols for + * endian.h. + */ +#if defined(__NetBSD__) && defined(_POSIX_C_SOURCE) +#undef _POSIX_C_SOURCE +#endif + #include "tree_sitter/api.h" #include "./alloc.h" #include "./array.h" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/stack.c new/tree-sitter-0.25.3/lib/src/stack.c --- old/tree-sitter-0.25.1/lib/src/stack.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/stack.c 2025-03-05 01:03:16.000000000 +0100 @@ -460,6 +460,17 @@ return self->heads.size; } +uint32_t ts_stack_halted_version_count(Stack *self) { + uint32_t count = 0; + for (uint32_t i = 0; i < self->heads.size; i++) { + StackHead *head = array_get(&self->heads, i); + if (head->status == StackStatusHalted) { + count++; + } + } + return count; +} + TSStateId ts_stack_state(const Stack *self, StackVersion version) { return array_get(&self->heads, version)->node->state; } @@ -524,6 +535,7 @@ return stack__iter(self, version, pop_count_callback, &count, (int)count); } + forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) { (void)payload; if (iterator->subtree_count >= 1) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/stack.h new/tree-sitter-0.25.3/lib/src/stack.h --- old/tree-sitter-0.25.1/lib/src/stack.h 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/stack.h 2025-03-05 01:03:16.000000000 +0100 @@ -36,6 +36,9 @@ // Get the stack's current number of versions. uint32_t ts_stack_version_count(const Stack *self); +// Get the stack's current number of halted versions. +uint32_t ts_stack_halted_version_count(Stack *self); + // Get the state at the top of the given version of the stack. If the stack is // empty, this returns the initial state, 0. TSStateId ts_stack_state(const Stack *self, StackVersion version); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/tree_cursor.c new/tree-sitter-0.25.3/lib/src/tree_cursor.c --- old/tree-sitter-0.25.1/lib/src/tree_cursor.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/tree_cursor.c 2025-03-05 01:03:16.000000000 +0100 @@ -304,8 +304,9 @@ } TreeCursorStep ts_tree_cursor_goto_sibling_internal( - TSTreeCursor *_self, - bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)) { + TSTreeCursor *_self, + bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *) +) { TreeCursor *self = (TreeCursor *)_self; uint32_t initial_size = self->stack.size; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/lib/src/wasm_store.c new/tree-sitter-0.25.3/lib/src/wasm_store.c --- old/tree-sitter-0.25.1/lib/src/wasm_store.c 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/lib/src/wasm_store.c 2025-03-05 01:03:16.000000000 +0100 @@ -1220,6 +1220,10 @@ const uint8_t *memory = wasmtime_memory_data(context, &self->memory); memcpy(&wasm_language, &memory[language_address], sizeof(LanguageInWasmMemory)); + bool has_supertypes = + wasm_language.abi_version > LANGUAGE_VERSION_WITH_RESERVED_WORDS && + wasm_language.supertype_count > 0; + int32_t addresses[] = { wasm_language.parse_table, wasm_language.small_parse_table, @@ -1239,9 +1243,9 @@ wasm_language.primary_state_ids, wasm_language.name, wasm_language.reserved_words, - wasm_language.supertype_symbols, - wasm_language.supertype_map_entries, - wasm_language.supertype_map_slices, + has_supertypes ? wasm_language.supertype_symbols : 0, + has_supertypes ? wasm_language.supertype_map_entries : 0, + has_supertypes ? wasm_language.supertype_map_slices : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.states : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.symbol_map : 0, wasm_language.external_token_count > 0 ? wasm_language.external_scanner.create : 0, @@ -1331,7 +1335,7 @@ ); } - if (language->supertype_count > 0) { + if (has_supertypes) { language->supertype_symbols = copy( &memory[wasm_language.supertype_symbols], wasm_language.supertype_count * sizeof(TSSymbol) @@ -1609,13 +1613,22 @@ } } +// The data fields of TSLexer, without the function pointers. +// +// This portion of the struct needs to be copied in and out +// of wasm memory before and after calling a scan function. +typedef struct { + int32_t lookahead; + TSSymbol result_symbol; +} TSLexerDataPrefix; + static bool ts_wasm_store__call_lex_function(TSWasmStore *self, unsigned function_index, TSStateId state) { wasmtime_context_t *context = wasmtime_store_context(self->store); uint8_t *memory_data = wasmtime_memory_data(context, &self->memory); memcpy( &memory_data[self->lexer_address], - &self->current_lexer->lookahead, - sizeof(self->current_lexer->lookahead) + self->current_lexer, + sizeof(TSLexerDataPrefix) ); wasmtime_val_raw_t args[2] = { @@ -1627,9 +1640,9 @@ bool result = args[0].i32; memcpy( - &self->current_lexer->lookahead, + self->current_lexer, &memory_data[self->lexer_address], - sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol) + sizeof(TSLexerDataPrefix) ); return result; } @@ -1674,8 +1687,8 @@ memcpy( &memory_data[self->lexer_address], - &self->current_lexer->lookahead, - sizeof(self->current_lexer->lookahead) + self->current_lexer, + sizeof(TSLexerDataPrefix) ); uint32_t valid_tokens_address = @@ -1690,9 +1703,9 @@ if (self->has_error) return false; memcpy( - &self->current_lexer->lookahead, + self->current_lexer, &memory_data[self->lexer_address], - sizeof(self->current_lexer->lookahead) + sizeof(self->current_lexer->result_symbol) + sizeof(TSLexerDataPrefix) ); return args[0].i32; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt --- old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt 1970-01-01 01:00:00.000000000 +0100 +++ new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/corpus.txt 2025-03-05 01:03:16.000000000 +0100 @@ -0,0 +1,9 @@ +========================== +A document +========================== + +a b + +--- + +(document) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js --- old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js 1970-01-01 01:00:00.000000000 +0100 +++ new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/grammar.js 2025-03-05 01:03:16.000000000 +0100 @@ -0,0 +1,11 @@ +module.exports = grammar({ + name: 'epsilon_external_extra_tokens', + + extras: $ => [/\s/, $.comment], + + externals: $ => [$.comment], + + rules: { + document: $ => seq('a', 'b'), + } +}); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c --- old/tree-sitter-0.25.1/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c 1970-01-01 01:00:00.000000000 +0100 +++ new/tree-sitter-0.25.3/test/fixtures/test_grammars/epsilon_external_extra_tokens/scanner.c 2025-03-05 01:03:16.000000000 +0100 @@ -0,0 +1,33 @@ +#include "tree_sitter/parser.h" + +enum TokenType { + COMMENT +}; + +void *tree_sitter_epsilon_external_extra_tokens_external_scanner_create(void) { + return NULL; +} + +bool tree_sitter_epsilon_external_extra_tokens_external_scanner_scan( + void *payload, + TSLexer *lexer, + const bool *valid_symbols +) { + lexer->result_symbol = COMMENT; + return true; +} + +unsigned tree_sitter_epsilon_external_extra_tokens_external_scanner_serialize( + void *payload, + char *buffer +) { + return 0; +} + +void tree_sitter_epsilon_external_extra_tokens_external_scanner_deserialize( + void *payload, + const char *buffer, + unsigned length +) {} + +void tree_sitter_epsilon_external_extra_tokens_external_scanner_destroy(void *payload) {} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/tree-sitter-0.25.1/xtask/src/main.rs new/tree-sitter-0.25.3/xtask/src/main.rs --- old/tree-sitter-0.25.1/xtask/src/main.rs 2025-02-02 08:07:36.000000000 +0100 +++ new/tree-sitter-0.25.3/xtask/src/main.rs 2025-03-05 01:03:16.000000000 +0100 @@ -144,7 +144,7 @@ iterations: Option<u32>, /// Set the seed used to control random behavior. #[arg(long, short)] - seed: Option<u32>, + seed: Option<usize>, /// Print parsing log to stderr. #[arg(long, short)] debug: bool, ++++++ vendor.tar.zst ++++++ /work/SRC/openSUSE:Factory/tree-sitter/vendor.tar.zst /work/SRC/openSUSE:Factory/.tree-sitter.new.1907/vendor.tar.zst differ: char 7, line 1
