This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new ef732be0998 SOLR-18523: Remove the checkJavadocLinks crawl from the
documentation build (#5062)
ef732be0998 is described below
commit ef732be0998865aab635db323ca98a6d15d5bb2c
Author: Nick <[email protected]>
AuthorDate: Fri Oct 16 18:11:15 2026 -0600
SOLR-18523: Remove the checkJavadocLinks crawl from the documentation build
(#5062)
Mostly redundant with javadoc "DocLint", which we do.
---
build.gradle | 1 -
changelog/unreleased/SOLR-18523.yml | 7 +
dev-tools/scripts/checkJavadocLinks.py | 277 ----------------------------
gradle/documentation/documentation.gradle | 17 ++
gradle/validation/check-broken-links.gradle | 81 --------
5 files changed, 24 insertions(+), 359 deletions(-)
diff --git a/build.gradle b/build.gradle
index 35961bc3e10..5b0a550e0be 100644
--- a/build.gradle
+++ b/build.gradle
@@ -178,7 +178,6 @@ apply from:
file('gradle/validation/dependency-analyze.gradle.kts')
apply from: file('gradle/validation/ecj-lint.gradle')
apply from: file('gradle/validation/gradlew-scripts-tweaked.gradle')
apply from: file('gradle/validation/validate-log-calls.gradle')
-apply from: file('gradle/validation/check-broken-links.gradle')
apply from: file('gradle/validation/solr.config-file-sanity.gradle')
apply from: file('gradle/validation/dependencies.gradle')
diff --git a/changelog/unreleased/SOLR-18523.yml
b/changelog/unreleased/SOLR-18523.yml
new file mode 100644
index 00000000000..16a1d472154
--- /dev/null
+++ b/changelog/unreleased/SOLR-18523.yml
@@ -0,0 +1,7 @@
+title: "Remove the checkJavadocLinks crawl from the documentation build"
+type: other
+authors:
+ - name: Nick Shanin
+links:
+ - name: SOLR-18523
+ url: https://issues.apache.org/jira/browse/SOLR-18523
diff --git a/dev-tools/scripts/checkJavadocLinks.py
b/dev-tools/scripts/checkJavadocLinks.py
deleted file mode 100644
index 56dffc8ac41..00000000000
--- a/dev-tools/scripts/checkJavadocLinks.py
+++ /dev/null
@@ -1,277 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import traceback
-import os
-import sys
-import re
-from html.parser import HTMLParser
-import urllib.parse as urlparse
-
-reHyperlink = re.compile(r'<a(\s+.*?)>', re.I)
-reAtt = re.compile(r"""(?:\s+([a-z]+)\s*=\s*("[^"]*"|'[^']?'|[^'"\s]+))+""",
re.I)
-
-# Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
[#x10000-#x10FFFF] /* any Unicode character, excluding the surrogate blocks,
FFFE, and FFFF. */
-reValidChar =
re.compile("^[^\u0000-\u0008\u000B-\u000C\u000E-\u001F\uFFFE\uFFFF]*$")
-
-# silly emacs: '
-
-class FindHyperlinks(HTMLParser):
-
- def __init__(self, baseURL):
- HTMLParser.__init__(self)
- self.stack = []
- self.anchors = set()
- self.links = []
- self.baseURL = baseURL
- self.printed = False
-
- def handle_starttag(self, tag, attrs):
- # NOTE: I don't think 'a' should be in here. But try debugging
- # NumericRangeQuery.html. (Could be javadocs bug, it's a generic type...)
- if tag not in ('link', 'meta', 'frame', 'br', 'wbr', 'hr', 'p', 'li',
'img', 'col', 'a', 'dt', 'dd', 'input'):
- self.stack.append(tag)
- if tag == 'a':
- id = None
- name = None
- href = None
- for attName, attValue in attrs:
- if attName == 'name':
- name = attValue
- elif attName == 'href':
- href = attValue
- elif attName == 'id':
- id = attValue
-
- if name is not None:
- assert href is None
- if name in self.anchors:
- if name in ('serializedForm',
- 'serialized_methods',
- 'readObject(java.io.ObjectInputStream)',
- 'writeObject(java.io.ObjectOutputStream)') \
- and self.baseURL.endswith('/serialized-form.html'):
- # Seems like a bug in Javadoc generation... you can't have
- # same anchor name more than once...
- pass
- else:
- self.printFile()
- raise RuntimeError('anchor "%s" appears more than once' % name)
- else:
- self.anchors.add(name)
- elif href is not None:
- assert name is None
- href = href.strip()
- self.links.append(urlparse.urljoin(self.baseURL, href))
- elif id is None:
- raise RuntimeError('couldn\'t find an href nor name in link in %s:
only got these attrs: %s' % (self.baseURL, attrs))
-
- def handle_endtag(self, tag):
- if tag in ('link', 'meta', 'frame', 'br', 'hr', 'p', 'li', 'img', 'col',
'a', 'dt', 'dd', 'input'):
- return
-
- if len(self.stack) == 0:
- raise RuntimeError('%s %s:%s: saw </%s> no opening <%s>' %
(self.baseURL, self.getpos()[0], self.getpos()[1], tag, self.stack[-1]))
-
- if self.stack[-1] == tag:
- self.stack.pop()
- else:
- raise RuntimeError('%s %s:%s: saw </%s> but expected </%s>' %
(self.baseURL, self.getpos()[0], self.getpos()[1], tag, self.stack[-1]))
-
- def printFile(self):
- if not self.printed:
- print()
- print(' ' + self.baseURL)
- self.printed = True
-
-def parse(baseURL, html):
- global failures
- # look for broken unicode
- if not reValidChar.match(html):
- print(' WARNING: invalid characters detected in: %s' % baseURL)
- failures = True
- return [], []
-
- parser = FindHyperlinks(baseURL)
- try:
- parser.feed(html)
- parser.close()
- except:
- # TODO: Python's html.parser is now always lenient, which is no good for
us: we want correct HTML in our javadocs
- parser.printFile()
- print(' WARNING: failed to parse %s:' % baseURL)
- traceback.print_exc(file=sys.stdout)
- failures = True
- return [], []
-
- #print ' %d links, %d anchors' % \
- # (len(parser.links), len(parser.anchors))
- return parser.links, parser.anchors
-
-failures = False
-
-def checkAll(dirName):
- """
- Checks *.html (recursively) under this directory.
- """
-
- global failures
-
- # Find/parse all HTML files first
- print()
- print('Crawl/parse...')
- allFiles = {}
-
- if os.path.isfile(dirName):
- root, fileName = os.path.split(dirName)
- iter = ((root, [], [fileName]),)
- else:
- iter = os.walk(dirName)
-
- for root, dirs, files in iter:
- for f in files:
- main, ext = os.path.splitext(f)
- ext = ext.lower()
-
- # maybe?:
- # and main not in ('serialized-form'):
- if ext in ('.htm', '.html') and \
- not f.startswith('.#') and \
- main not in ('deprecated-list',):
- # Somehow even w/ java 7 generaged javadocs,
- # deprecated-list.html can fail to escape generics types
- fullPath = os.path.join(root, f).replace(os.path.sep,'/')
- fullPath = 'file:%s' % urlparse.quote(fullPath)
- # parse and unparse the URL to "normalize" it
- fullPath = urlparse.urlunparse(urlparse.urlparse(fullPath))
- #print ' %s' % fullPath
- allFiles[fullPath] = parse(fullPath, open('%s/%s' % (root, f),
encoding='UTF-8').read())
-
- # ... then verify:
- print()
- print('Verify...')
- for fullPath, (links, anchors) in allFiles.items():
- #print fullPath
- printed = False
- for link in links:
-
- origLink = link
-
- # TODO: use urlparse?
- idx = link.find('#')
- if idx != -1:
- anchor = link[idx+1:]
- link = link[:idx]
- else:
- anchor = None
-
- # remove any whitespace from the middle of the link
- link = ''.join(link.split())
-
- idx = link.find('?')
- if idx != -1:
- link = link[:idx]
-
- # TODO: normalize path sep for windows...
- if link.startswith('http://') or link.startswith('https://'):
- # don't check external links
-
- if link.find('lucene.apache.org/java/docs/mailinglists.html') != -1:
- # OK
- pass
- elif link == 'http://lucene.apache.org/core/':
- # OK
- pass
- elif re.match("^https?://lucene.apache.org/core/[^/]+/index.html",
link):
- # OK
- pass
- elif link == 'http://lucene.apache.org/solr/':
- # OK
- pass
- elif link == 'http://lucene.apache.org/solr/resources.html':
- # OK
- pass
- elif link.find('lucene.apache.org/java/docs/discussion.html') != -1:
- # OK
- pass
- elif link.find('lucene.apache.org/core/discussion.html') != -1:
- # OK
- pass
- elif
link.find('lucene.apache.org/solr/mirrors-solr-latest-redir.html') != -1:
- # OK
- pass
- elif link.find('lucene.apache.org/solr/guide/') != -1:
- # OK
- pass
- elif link.find('lucene.apache.org/solr/downloads.html') != -1:
- # OK
- pass
- elif (link.find('svn.apache.org') != -1
- or link.find('lucene.apache.org') != -1)\
- and os.path.basename(fullPath) != 'Changes.html':
- if not printed:
- printed = True
- print()
- print(fullPath)
- print(' BAD EXTERNAL LINK: %s' % link)
- elif link.startswith('mailto:'):
- if link.find('@lucene.apache.org') == -1 and link.find('@apache.org')
!= -1:
- if not printed:
- printed = True
- print()
- print(fullPath)
- print(' BROKEN MAILTO (?): %s' % link)
- elif link.startswith('javascript:'):
- # ok...?
- pass
- elif 'org/apache/solr/client/solrj/beans/Field.html' in link:
- # see LUCENE-4011: this is a javadocs bug for constants
- # on annotations it seems?
- pass
- elif link.startswith('file:'):
- if link not in allFiles:
- filepath = urlparse.unquote(urlparse.urlparse(link).path)
- if not (os.path.exists(filepath) or os.path.exists(filepath[1:])):
- if not printed:
- printed = True
- print()
- print(fullPath)
- print(' BROKEN LINK: %s' % link)
- elif anchor is not None and anchor not in allFiles[link][1]:
- if not printed:
- printed = True
- print()
- print(fullPath)
- print(' BROKEN ANCHOR: %s' % origLink)
- else:
- if not printed:
- printed = True
- print()
- print(fullPath)
- print(' BROKEN URL SCHEME: %s' % origLink)
- failures = failures or printed
-
- return failures
-
-if __name__ == '__main__':
- if checkAll(sys.argv[1]):
- print()
- print('Broken javadocs links were found! Common root causes:')
- # please feel free to add to this list
- print('* A typo of some sort for manually created links.')
- print('* Public methods referencing non-public classes in their
signature.')
- sys.exit(1)
- sys.exit(0)
-
diff --git a/gradle/documentation/documentation.gradle
b/gradle/documentation/documentation.gradle
index 700116ea13d..c680fe25d54 100644
--- a/gradle/documentation/documentation.gradle
+++ b/gradle/documentation/documentation.gradle
@@ -71,6 +71,23 @@ configure(project(':solr:documentation')) {
'markdownToHtml', 'createDocumentationIndex'
}
+ // Javadoc references are validated at compile time by DocLint
+ // (gradle/java/javac.gradle). Building the full documentation is costly, so
+ // this project's check builds it only on CI, where it also exercises
+ // references that only the javadoc tool sees, such as those in overview
+ // files, and the missing-doclet checks. The gate covers this project's
+ // check path only; a root check still builds the site through the ref
+ // guide's checkSiteLinks task, when the ref guide is included
+ // (refguide.include, on by default when a .git directory exists).
+ //
+ // Overview references are checked when the render tasks run. overview.html
+ // is not a declared input of the render task, so in a reused workspace an
+ // overview-only change can leave the task up-to-date and unchecked. A
+ // fresh CI checkout runs it: the build cache is off on CI.
+ if (isCIBuild) {
+ check.dependsOn 'documentation'
+ }
+
task copyDocumentationAssets(type: Copy) {
includeEmptyDirs = false
from(project.assets)
diff --git a/gradle/validation/check-broken-links.gradle
b/gradle/validation/check-broken-links.gradle
deleted file mode 100644
index 66b426abcf7..00000000000
--- a/gradle/validation/check-broken-links.gradle
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.gradle.process.ExecOperations
-import javax.inject.Inject
-
-configure(rootProject) {
- task checkBrokenLinks {
- group 'Verification'
- description 'Check broken links in the entire documentation'
-
- dependsOn ':solr:documentation:checkBrokenLinks'
- }
-}
-
-configure(project(':solr:documentation')) {
- task checkBrokenLinks(type: CheckBrokenLinksTask, 'dependsOn':
'documentation')
- check.dependsOn checkBrokenLinks
-}
-
-@CacheableTask
-abstract class CheckBrokenLinksTask extends DefaultTask {
-
- @Inject
- abstract ExecOperations getExecOperations()
-
- // wraps input directory location in DirectoryProperty so as to lazily
evaluate 'docroot' property
- // (see gradle/documentation/documentation.gradle)
- @InputDirectory
- @PathSensitive(PathSensitivity.RELATIVE)
- final DirectoryProperty docsDir = project.objects.directoryProperty()
- .fileProvider(project.providers.provider {project.docroot})
-
- @InputFile
- @PathSensitive(PathSensitivity.RELATIVE)
- File script =
project.rootProject.file("dev-tools/scripts/checkJavadocLinks.py")
-
- @OutputFile
- Provider<RegularFile> outputFile = project.layout.buildDirectory.file(
- 'check-broken-links-output.txt')
-
- @TaskAction
- def check() {
- def result
- outputFile.get().asFile.withOutputStream {output ->
- result = execOperations.exec {
- executable project.externalTool("python3")
- ignoreExitValue = true
- standardOutput = output
- errorOutput = output
- args = [
- "-B",
- script.absolutePath,
- docsDir.get().getAsFile()
- ]
- }
- }
-
- if (result.getExitValue() != 0) {
- println "Broken links check failed. First lines of the output:"
- outputFile.get().asFile.readLines().take(20).each {line ->
- println line
- }
- throw new GradleException("Broken links check failed. Command output at:
${outputFile.get()}")
- }
- }
-}
