This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit 9eea9ee6205019105b258815f9c02c4268a4f998 Author: Nick <[email protected]> AuthorDate: Fri Oct 16 18:11:15 2026 -0600 SOLR-18523: Remove the checkJavadocLinks crawl from the documentation build (#5062) Mostly redundant with javadoc "DocLint", which we do. (cherry picked from commit ef732be0998865aab635db323ca98a6d15d5bb2c) --- build.gradle | 1 - changelog/unreleased/SOLR-18523.yml | 7 + dev-tools/scripts/checkJavadocLinks.py | 277 ---------------------------- gradle/documentation/documentation.gradle | 17 ++ gradle/validation/check-broken-links.gradle | 81 -------- 5 files changed, 24 insertions(+), 359 deletions(-) diff --git a/build.gradle b/build.gradle index 11b8005ca2d..6bd4467abab 100644 --- a/build.gradle +++ b/build.gradle @@ -178,7 +178,6 @@ apply from: file('gradle/validation/dependency-analyze.gradle.kts') apply from: file('gradle/validation/ecj-lint.gradle') apply from: file('gradle/validation/gradlew-scripts-tweaked.gradle') apply from: file('gradle/validation/validate-log-calls.gradle') -apply from: file('gradle/validation/check-broken-links.gradle') apply from: file('gradle/validation/solr.config-file-sanity.gradle') apply from: file('gradle/validation/dependencies.gradle') diff --git a/changelog/unreleased/SOLR-18523.yml b/changelog/unreleased/SOLR-18523.yml new file mode 100644 index 00000000000..16a1d472154 --- /dev/null +++ b/changelog/unreleased/SOLR-18523.yml @@ -0,0 +1,7 @@ +title: "Remove the checkJavadocLinks crawl from the documentation build" +type: other +authors: + - name: Nick Shanin +links: + - name: SOLR-18523 + url: https://issues.apache.org/jira/browse/SOLR-18523 diff --git a/dev-tools/scripts/checkJavadocLinks.py b/dev-tools/scripts/checkJavadocLinks.py deleted file mode 100644 index 56dffc8ac41..00000000000 --- a/dev-tools/scripts/checkJavadocLinks.py +++ /dev/null @@ -1,277 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import traceback -import os -import sys -import re -from html.parser import HTMLParser -import urllib.parse as urlparse - -reHyperlink = re.compile(r'<a(\s+.*?)>', re.I) -reAtt = re.compile(r"""(?:\s+([a-z]+)\s*=\s*("[^"]*"|'[^']?'|[^'"\s]+))+""", re.I) - -# Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF. */ -reValidChar = re.compile("^[^\u0000-\u0008\u000B-\u000C\u000E-\u001F\uFFFE\uFFFF]*$") - -# silly emacs: ' - -class FindHyperlinks(HTMLParser): - - def __init__(self, baseURL): - HTMLParser.__init__(self) - self.stack = [] - self.anchors = set() - self.links = [] - self.baseURL = baseURL - self.printed = False - - def handle_starttag(self, tag, attrs): - # NOTE: I don't think 'a' should be in here. But try debugging - # NumericRangeQuery.html. (Could be javadocs bug, it's a generic type...) - if tag not in ('link', 'meta', 'frame', 'br', 'wbr', 'hr', 'p', 'li', 'img', 'col', 'a', 'dt', 'dd', 'input'): - self.stack.append(tag) - if tag == 'a': - id = None - name = None - href = None - for attName, attValue in attrs: - if attName == 'name': - name = attValue - elif attName == 'href': - href = attValue - elif attName == 'id': - id = attValue - - if name is not None: - assert href is None - if name in self.anchors: - if name in ('serializedForm', - 'serialized_methods', - 'readObject(java.io.ObjectInputStream)', - 'writeObject(java.io.ObjectOutputStream)') \ - and self.baseURL.endswith('/serialized-form.html'): - # Seems like a bug in Javadoc generation... you can't have - # same anchor name more than once... - pass - else: - self.printFile() - raise RuntimeError('anchor "%s" appears more than once' % name) - else: - self.anchors.add(name) - elif href is not None: - assert name is None - href = href.strip() - self.links.append(urlparse.urljoin(self.baseURL, href)) - elif id is None: - raise RuntimeError('couldn\'t find an href nor name in link in %s: only got these attrs: %s' % (self.baseURL, attrs)) - - def handle_endtag(self, tag): - if tag in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col', 'a', 'dt', 'dd', 'input'): - return - - if len(self.stack) == 0: - raise RuntimeError('%s %s:%s: saw </%s> no opening <%s>' % (self.baseURL, self.getpos()[0], self.getpos()[1], tag, self.stack[-1])) - - if self.stack[-1] == tag: - self.stack.pop() - else: - raise RuntimeError('%s %s:%s: saw </%s> but expected </%s>' % (self.baseURL, self.getpos()[0], self.getpos()[1], tag, self.stack[-1])) - - def printFile(self): - if not self.printed: - print() - print(' ' + self.baseURL) - self.printed = True - -def parse(baseURL, html): - global failures - # look for broken unicode - if not reValidChar.match(html): - print(' WARNING: invalid characters detected in: %s' % baseURL) - failures = True - return [], [] - - parser = FindHyperlinks(baseURL) - try: - parser.feed(html) - parser.close() - except: - # TODO: Python's html.parser is now always lenient, which is no good for us: we want correct HTML in our javadocs - parser.printFile() - print(' WARNING: failed to parse %s:' % baseURL) - traceback.print_exc(file=sys.stdout) - failures = True - return [], [] - - #print ' %d links, %d anchors' % \ - # (len(parser.links), len(parser.anchors)) - return parser.links, parser.anchors - -failures = False - -def checkAll(dirName): - """ - Checks *.html (recursively) under this directory. - """ - - global failures - - # Find/parse all HTML files first - print() - print('Crawl/parse...') - allFiles = {} - - if os.path.isfile(dirName): - root, fileName = os.path.split(dirName) - iter = ((root, [], [fileName]),) - else: - iter = os.walk(dirName) - - for root, dirs, files in iter: - for f in files: - main, ext = os.path.splitext(f) - ext = ext.lower() - - # maybe?: - # and main not in ('serialized-form'): - if ext in ('.htm', '.html') and \ - not f.startswith('.#') and \ - main not in ('deprecated-list',): - # Somehow even w/ java 7 generaged javadocs, - # deprecated-list.html can fail to escape generics types - fullPath = os.path.join(root, f).replace(os.path.sep,'/') - fullPath = 'file:%s' % urlparse.quote(fullPath) - # parse and unparse the URL to "normalize" it - fullPath = urlparse.urlunparse(urlparse.urlparse(fullPath)) - #print ' %s' % fullPath - allFiles[fullPath] = parse(fullPath, open('%s/%s' % (root, f), encoding='UTF-8').read()) - - # ... then verify: - print() - print('Verify...') - for fullPath, (links, anchors) in allFiles.items(): - #print fullPath - printed = False - for link in links: - - origLink = link - - # TODO: use urlparse? - idx = link.find('#') - if idx != -1: - anchor = link[idx+1:] - link = link[:idx] - else: - anchor = None - - # remove any whitespace from the middle of the link - link = ''.join(link.split()) - - idx = link.find('?') - if idx != -1: - link = link[:idx] - - # TODO: normalize path sep for windows... - if link.startswith('http://') or link.startswith('https://'): - # don't check external links - - if link.find('lucene.apache.org/java/docs/mailinglists.html') != -1: - # OK - pass - elif link == 'http://lucene.apache.org/core/': - # OK - pass - elif re.match("^https?://lucene.apache.org/core/[^/]+/index.html", link): - # OK - pass - elif link == 'http://lucene.apache.org/solr/': - # OK - pass - elif link == 'http://lucene.apache.org/solr/resources.html': - # OK - pass - elif link.find('lucene.apache.org/java/docs/discussion.html') != -1: - # OK - pass - elif link.find('lucene.apache.org/core/discussion.html') != -1: - # OK - pass - elif link.find('lucene.apache.org/solr/mirrors-solr-latest-redir.html') != -1: - # OK - pass - elif link.find('lucene.apache.org/solr/guide/') != -1: - # OK - pass - elif link.find('lucene.apache.org/solr/downloads.html') != -1: - # OK - pass - elif (link.find('svn.apache.org') != -1 - or link.find('lucene.apache.org') != -1)\ - and os.path.basename(fullPath) != 'Changes.html': - if not printed: - printed = True - print() - print(fullPath) - print(' BAD EXTERNAL LINK: %s' % link) - elif link.startswith('mailto:'): - if link.find('@lucene.apache.org') == -1 and link.find('@apache.org') != -1: - if not printed: - printed = True - print() - print(fullPath) - print(' BROKEN MAILTO (?): %s' % link) - elif link.startswith('javascript:'): - # ok...? - pass - elif 'org/apache/solr/client/solrj/beans/Field.html' in link: - # see LUCENE-4011: this is a javadocs bug for constants - # on annotations it seems? - pass - elif link.startswith('file:'): - if link not in allFiles: - filepath = urlparse.unquote(urlparse.urlparse(link).path) - if not (os.path.exists(filepath) or os.path.exists(filepath[1:])): - if not printed: - printed = True - print() - print(fullPath) - print(' BROKEN LINK: %s' % link) - elif anchor is not None and anchor not in allFiles[link][1]: - if not printed: - printed = True - print() - print(fullPath) - print(' BROKEN ANCHOR: %s' % origLink) - else: - if not printed: - printed = True - print() - print(fullPath) - print(' BROKEN URL SCHEME: %s' % origLink) - failures = failures or printed - - return failures - -if __name__ == '__main__': - if checkAll(sys.argv[1]): - print() - print('Broken javadocs links were found! Common root causes:') - # please feel free to add to this list - print('* A typo of some sort for manually created links.') - print('* Public methods referencing non-public classes in their signature.') - sys.exit(1) - sys.exit(0) - diff --git a/gradle/documentation/documentation.gradle b/gradle/documentation/documentation.gradle index 700116ea13d..c680fe25d54 100644 --- a/gradle/documentation/documentation.gradle +++ b/gradle/documentation/documentation.gradle @@ -71,6 +71,23 @@ configure(project(':solr:documentation')) { 'markdownToHtml', 'createDocumentationIndex' } + // Javadoc references are validated at compile time by DocLint + // (gradle/java/javac.gradle). Building the full documentation is costly, so + // this project's check builds it only on CI, where it also exercises + // references that only the javadoc tool sees, such as those in overview + // files, and the missing-doclet checks. The gate covers this project's + // check path only; a root check still builds the site through the ref + // guide's checkSiteLinks task, when the ref guide is included + // (refguide.include, on by default when a .git directory exists). + // + // Overview references are checked when the render tasks run. overview.html + // is not a declared input of the render task, so in a reused workspace an + // overview-only change can leave the task up-to-date and unchecked. A + // fresh CI checkout runs it: the build cache is off on CI. + if (isCIBuild) { + check.dependsOn 'documentation' + } + task copyDocumentationAssets(type: Copy) { includeEmptyDirs = false from(project.assets) diff --git a/gradle/validation/check-broken-links.gradle b/gradle/validation/check-broken-links.gradle deleted file mode 100644 index 66b426abcf7..00000000000 --- a/gradle/validation/check-broken-links.gradle +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.gradle.process.ExecOperations -import javax.inject.Inject - -configure(rootProject) { - task checkBrokenLinks { - group 'Verification' - description 'Check broken links in the entire documentation' - - dependsOn ':solr:documentation:checkBrokenLinks' - } -} - -configure(project(':solr:documentation')) { - task checkBrokenLinks(type: CheckBrokenLinksTask, 'dependsOn': 'documentation') - check.dependsOn checkBrokenLinks -} - -@CacheableTask -abstract class CheckBrokenLinksTask extends DefaultTask { - - @Inject - abstract ExecOperations getExecOperations() - - // wraps input directory location in DirectoryProperty so as to lazily evaluate 'docroot' property - // (see gradle/documentation/documentation.gradle) - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - final DirectoryProperty docsDir = project.objects.directoryProperty() - .fileProvider(project.providers.provider {project.docroot}) - - @InputFile - @PathSensitive(PathSensitivity.RELATIVE) - File script = project.rootProject.file("dev-tools/scripts/checkJavadocLinks.py") - - @OutputFile - Provider<RegularFile> outputFile = project.layout.buildDirectory.file( - 'check-broken-links-output.txt') - - @TaskAction - def check() { - def result - outputFile.get().asFile.withOutputStream {output -> - result = execOperations.exec { - executable project.externalTool("python3") - ignoreExitValue = true - standardOutput = output - errorOutput = output - args = [ - "-B", - script.absolutePath, - docsDir.get().getAsFile() - ] - } - } - - if (result.getExitValue() != 0) { - println "Broken links check failed. First lines of the output:" - outputFile.get().asFile.readLines().take(20).each {line -> - println line - } - throw new GradleException("Broken links check failed. Command output at: ${outputFile.get()}") - } - } -}
