llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-clang-tools-extra Author: Cyrus Ding (dingcyrus) <details> <summary>Changes</summary> When a source file comment contains non-UTF-8 bytes (e.g., byte 0x97), clang-doc's JSONGenerator would crash with an assertion failure: Assertion `false && "Invalid UTF-8 in value used as JSON"' failed. This happens because json::Value(StringRef) asserts on invalid UTF-8 input in debug builds. While the release-build path calls fixUTF8() automatically, the assertion crash affects developers and fuzzer runs. Fix by following the same pattern used elsewhere in LLVM (lldb, clangd): check isUTF8() before constructing a json::Value, and call fixUTF8() to sanitize the string if invalid bytes are detected. This avoids an unnecessary copy for the common case of valid UTF-8. The fix covers all paths in serializeComment() where user-written text from source comments enters JSON objects: - CK_TextComment (I.Text) - CK_InlineCommandComment (I.Args) - CK_VerbatimBlockLineComment / CK_VerbatimLineComment (I.Text) - CK_Unknown (I.Text) Also add a regression test with an actual invalid UTF-8 byte in a comment. AI assistance was used for code review analysis and local build verification. Fixes: https://github.com/llvm/llvm-project/issues/210675 --- Full diff: https://github.com/llvm/llvm-project/pull/210886.diff 3 Files Affected: - (modified) clang-tools-extra/clang-doc/JSONGenerator.cpp (+20-6) - (added) clang-tools-extra/test/clang-doc/Inputs/invalid-utf8-comment.cpp (+2) - (added) clang-tools-extra/test/clang-doc/json/invalid-utf8-comment.cpp (+50) ``````````diff diff --git a/clang-tools-extra/clang-doc/JSONGenerator.cpp b/clang-tools-extra/clang-doc/JSONGenerator.cpp index 58995c249bd21..eeedd372d8eb0 100644 --- a/clang-tools-extra/clang-doc/JSONGenerator.cpp +++ b/clang-tools-extra/clang-doc/JSONGenerator.cpp @@ -232,8 +232,12 @@ static Object serializeComment(const CommentInfo &I, Object &Description) { switch (I.Kind) { case CommentKind::CK_TextComment: { - if (!I.Text.empty()) - Obj.insert({commentKindToString(I.Kind), I.Text}); + if (!I.Text.empty()) { + if (LLVM_LIKELY(json::isUTF8(I.Text))) + Obj.insert({commentKindToString(I.Kind), I.Text}); + else + Obj.insert({commentKindToString(I.Kind), json::fixUTF8(I.Text)}); + } return Obj; } @@ -257,8 +261,12 @@ static Object serializeComment(const CommentInfo &I, Object &Description) { json::Value ArgsArr = Array(); auto &ARef = *ArgsArr.getAsArray(); ARef.reserve(I.Args.size()); - for (const auto &Arg : I.Args) - ARef.emplace_back(Arg); + for (const auto &Arg : I.Args) { + if (LLVM_LIKELY(json::isUTF8(Arg))) + ARef.emplace_back(Arg); + else + ARef.emplace_back(json::fixUTF8(Arg)); + } Child.insert({"Command", I.Name}); Child.insert({"Args", ArgsArr}); Child.insert({"Children", ChildArr}); @@ -292,7 +300,10 @@ static Object serializeComment(const CommentInfo &I, Object &Description) { case CommentKind::CK_VerbatimBlockLineComment: case CommentKind::CK_VerbatimLineComment: { - Child.insert({"Text", I.Text}); + if (LLVM_LIKELY(json::isUTF8(I.Text))) + Child.insert({"Text", I.Text}); + else + Child.insert({"Text", json::fixUTF8(I.Text)}); Child.insert({"Children", ChildArr}); Obj.insert({commentKindToString(I.Kind), ChildVal}); return Obj; @@ -333,7 +344,10 @@ static Object serializeComment(const CommentInfo &I, Object &Description) { } case CommentKind::CK_Unknown: { - Obj.insert({commentKindToString(I.Kind), I.Text}); + if (LLVM_LIKELY(json::isUTF8(I.Text))) + Obj.insert({commentKindToString(I.Kind), I.Text}); + else + Obj.insert({commentKindToString(I.Kind), json::fixUTF8(I.Text)}); return Obj; } } diff --git a/clang-tools-extra/test/clang-doc/Inputs/invalid-utf8-comment.cpp b/clang-tools-extra/test/clang-doc/Inputs/invalid-utf8-comment.cpp new file mode 100644 index 0000000000000..eb61805297c98 --- /dev/null +++ b/clang-tools-extra/test/clang-doc/Inputs/invalid-utf8-comment.cpp @@ -0,0 +1,2 @@ +// bad: � +class A {}; diff --git a/clang-tools-extra/test/clang-doc/json/invalid-utf8-comment.cpp b/clang-tools-extra/test/clang-doc/json/invalid-utf8-comment.cpp new file mode 100644 index 0000000000000..4e4f7f1b51860 --- /dev/null +++ b/clang-tools-extra/test/clang-doc/json/invalid-utf8-comment.cpp @@ -0,0 +1,50 @@ +// RUN: rm -rf %t && mkdir -p %t +// RUN: clang-doc --output=%t --format=json --executor=standalone %S/../Inputs/invalid-utf8-comment.cpp 2>&1 +// RUN: cat %t/json/GlobalNamespace/_ZTV1A.json | FileCheck %s + +// Regression test for https://github.com/llvm/llvm-project/issues/210675. +// clang-doc should not crash when encountering comments with invalid UTF-8 bytes. +// Instead, the invalid bytes should be replaced with valid UTF-8 replacement characters. + +// CHECK: { +// CHECK-NEXT: "Contexts": [ +// CHECK-NEXT: { +// CHECK-NEXT: "DocumentationFileName": "index", +// CHECK-NEXT: "End": true, +// CHECK-NEXT: "Name": "Global Namespace", +// CHECK-NEXT: "QualName": "GlobalNamespace", +// CHECK-NEXT: "RelativePath": "./", +// CHECK-NEXT: "USR": "0000000000000000000000000000000000000000" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "Description": { +// CHECK-NEXT: "BriefComments": [ +// CHECK-NEXT: [ +// CHECK-NEXT: { +// CHECK: "TextComment": +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: ], +// CHECK-NEXT: "HasBriefComments": true, +// CHECK-NEXT: "HasParagraphComments": true, +// CHECK-NEXT: "ParagraphComments": [ +// CHECK-NEXT: [ +// CHECK-NEXT: { +// CHECK: "TextComment": +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: ] +// CHECK-NEXT: }, +// CHECK-NEXT: "HasContexts": true, +// CHECK-NEXT: "InfoType": "record", +// CHECK-NEXT: "IsTypedef": false, +// CHECK: "Location": { +// CHECK: "MangledName": "_ZTV1A", +// CHECK-NEXT: "Name": "A", +// CHECK-NEXT: "Namespace": [ +// CHECK-NEXT: "GlobalNamespace" +// CHECK-NEXT: ], +// CHECK-NEXT: "Path": "GlobalNamespace", +// CHECK-NEXT: "TagType": "class", +// CHECK-NEXT: "USR": "{{[0-9A-F]*}}" +// CHECK-NEXT: } `````````` </details> https://github.com/llvm/llvm-project/pull/210886 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
