This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 1f652f6 SOLR-15436: Gradle's RAT task has missing inputs, so it can't
figure out when to run (#272)
1f652f6 is described below
commit 1f652f699c15aca3f85a6375bfd66a13a8565f66
Author: Dawid Weiss <[email protected]>
AuthorDate: Thu Aug 26 15:05:56 2021 +0200
SOLR-15436: Gradle's RAT task has missing inputs, so it can't figure out
when to run (#272)
---
gradle/validation/rat-sources.gradle | 241 +++++++++++++++++++++--------------
1 file changed, 148 insertions(+), 93 deletions(-)
diff --git a/gradle/validation/rat-sources.gradle
b/gradle/validation/rat-sources.gradle
index 41a6e78..9acd0dc 100644
--- a/gradle/validation/rat-sources.gradle
+++ b/gradle/validation/rat-sources.gradle
@@ -17,6 +17,7 @@
import groovy.xml.NamespaceBuilder
+// Configure rat dependencies for use in the custom task.
configure(rootProject) {
configurations {
ratDeps
@@ -27,111 +28,164 @@ configure(rootProject) {
}
}
+// Configure the rat validation task and all scanned directories.
allprojects {
task("rat", type: RatTask) {
group = 'Verification'
description = 'Runs Apache Rat checks.'
- }
-}
-configure(rootProject) {
- rat {
- includes += [
- "buildSrc/**/*.java",
- "gradle/**/*.gradle",
- "lucene/tools/forbiddenApis/**",
- "lucene/tools/prettify/**",
- ]
- excludes += [
- // Unclear if this needs ASF header, depends on how much was
copied from ElasticSearch
- "**/ErrorReportingTestListener.java"
- ]
- }
-}
+ def defaultScanFileTree = project.fileTree(projectDir, {
+ // Don't check under the project's build folder.
+ exclude project.buildDir.name
-configure(project(":solr:core")) {
- rat {
- srcExcludes += [
- "**/htmlStripReaderTest.html"
- ]
- }
-}
+ // Exclude any generated stuff.
+ exclude "src/generated"
+
+ // Don't check any of the subprojects - they have their own rat
tasks.
+ exclude subprojects.collect { it.projectDir.name }
+
+ // At the module scope we only check selected file patterns as
folks have various .gitignore-d resources
+ // generated by IDEs, etc.
+ include "**/*.gradle"
+ include "**/*.xml"
+ include "**/*.md"
+ include "**/*.py"
+ include "**/*.sh"
+ include "**/*.bat"
+
+ // Include selected patterns from any source folders. We could
make this
+ // relative to source sets but it seems to be of little value -
all our source sets
+ // live under 'src' anyway.
+ include "src/**"
+ exclude "src/**/*.png"
+ exclude "src/**/*.txt"
+ exclude "src/**/*.zip"
+ exclude "src/**/*.properties"
+ exclude "src/**/*.utf8"
+ exclude "src/**/*.svg"
+ exclude "src/**/*.csv"
+
+ // TODO: SOLR-15601: Some of these should carry the license,
perhaps?
+ exclude "**/README.md"
+ exclude "**/*.html"
+ exclude "**/*.xml"
+ exclude "**/*.json"
+ exclude "**/*.js"
+ exclude "**/*.py"
+
+ // Conditionally apply module-specific patterns. We do it here
instead
+ // of reconfiguring each project because the provider can be made
lazy
+ // and it's easier to manage this way.
+ switch (project.path) {
+ case ":":
+ include "gradlew"
+ include "gradlew.bat"
+ exclude ".gradle"
+ exclude ".idea"
+ exclude ".muse"
+ exclude ".git"
+
+ // Exclude github stuff (templates, workflows).
+ exclude ".github"
+
+ exclude "dev-tools/scripts/cloud.sh"
+ exclude "dev-tools/scripts/README.md"
+ exclude "dev-tools/scripts/create_line_file_docs.py"
-configure(project(":solr:webapp")) {
- rat {
- includes = ["**"]
- excludes += [
- "web/img/**",
- "*.iml",
- "build.gradle",
- "build/**",
- ]
+ // The root project also includes patterns for the
boostrap (buildSrc) and composite
+ // projects. Include their sources in the scan.
+ include "buildSrc/src/**"
+ include "dev-tools/missing-doclet/src/**"
+ break
+
+ case ":solr:contrib:clustering":
+ exclude "src/test-files/META-INF/services/*"
+ break
+
+ case ":solr:contrib:langid":
+ exclude "**/langdetect-profiles/*"
+ break
+
+ case ":solr:documentation":
+ exclude "src/markdown/*.md"
+ break
+
+ case ":solr:core":
+ exclude "**/htmlStripReaderTest.html"
+ exclude "src/resources/*.json"
+ exclude "src/resources/*.xml"
+ exclude "src/test-files/**/*.csv"
+ exclude "src/test-files/**/*.json"
+ exclude "src/test-files/**/*.aff"
+ exclude "src/test-files/**/*.dic"
+ exclude "src/test-files/**/*.conf"
+ exclude "src/test-files/**/external_eff"
+ exclude "src/test-files/**/*.incl"
+ break
+
+ case ":solr:server":
+ exclude "**/*.xml"
+ exclude "**/*.sh"
+ exclude "**/*.bat"
+ break
+
+ case ":solr:webapp":
+ exclude "web/img/**"
+ break
+
+ case ":solr:solr-ref-guide":
+ exclude "src/**"
+ break
+
+ case ":solr:docker":
+ exclude "tests/**/*.xml"
+ break
+
+ case ":solr:example":
+ exclude "**/*.xml"
+ exclude "films/README.md"
+ break
+
+ case ":solr:solrj":
+ exclude "src/**/*.json"
+ exclude "src/test-files/**/*.cfg"
+ exclude "src/test-files/**/*.xml"
+ break
+ }
+ })
+ inputFileTrees.add(defaultScanFileTree)
}
}
-// Structure inspired by existing task from Apache Kafka, heavily modified
since then.
+/**
+ * An Apache RAT adapter that validates whether files contain acceptable
licenses.
+ */
class RatTask extends DefaultTask {
- @Input
- List<String> includes = [
- "*.gradle",
- "*.xml",
- "src/tools/**"
- ]
-
- @Input
- List<String> excludes = []
-
- @Input
- List<String> srcExcludes = [
- "**/TODO",
- "**/*.txt",
- "**/*.md",
- "**/*.iml",
- "build/**"
- ]
+ @InputFiles
+ final ListProperty<ConfigurableFileTree> inputFileTrees =
project.objects.listProperty(ConfigurableFileTree)
@OutputFile
- def xmlReport = new File(new File(project.buildDir, 'rat'),
'rat-report.xml')
+ final RegularFileProperty xmlReport =
project.objects.fileProperty().convention(
+ project.layout.buildDirectory.file("rat/rat-report.xml"))
- def generateXmlReport() {
- def uri = 'antlib:org.apache.rat.anttasks'
+ def generateReport(File reportFile) {
+ // Set up ant rat task.
def ratClasspath = project.rootProject.configurations.ratDeps.asPath
- ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', uri: uri,
classpath: ratClasspath)
+ ant.taskdef(resource: 'org/apache/rat/anttasks/antlib.xml', classpath:
ratClasspath)
- def rat = NamespaceBuilder.newInstance(ant, uri)
- rat.report(format: 'xml', reportFile: xmlReport,
addDefaultLicenseMatchers: true) {
- ant.fileset(dir: "${project.projectDir}") {
- includes.each { pattern -> ant.include(name: pattern) }
- excludes.each { pattern -> ant.exclude(name: pattern) }
- }
+ // Collect all output files for debugging.
+ String inputFileList = inputFileTrees.get().collectMany { fileTree ->
+ fileTree.asList()
+ }.sort().join("\n")
+ project.file(reportFile.path.replaceAll('.xml$',
'-filelist.txt')).setText(inputFileList, "UTF-8")
- if (project.plugins.findPlugin(JavaPlugin)) {
- def checkSets = [
- project.sourceSets.main.java.srcDirs,
- project.sourceSets.test.java.srcDirs,
- ]
-
- project.sourceSets.matching { it.name == 'tools' }.all {
- checkSets += project.sourceSets.tools.java.srcDirs
- }
-
- checkSets.flatten().each { srcLocation ->
- ant.fileset(dir: srcLocation, erroronmissingdir: false) {
- srcExcludes.each { pattern -> ant.exclude(name:
pattern) }
- }
- }
-
- [
- project.sourceSets.main.resources.srcDirs
- ].flatten().each { srcLocation ->
- ant.fileset(dir: srcLocation, erroronmissingdir: false) {
- ant.include(name: "META-INF/**")
- }
- }
+ // Run rat via ant.
+ ant.report(format: 'xml', reportFile: reportFile,
addDefaultLicenseMatchers: true) {
+ // Pass all gradle file trees to the ant task (Gradle's internal
adapters are used).
+ inputFileTrees.get().each { fileTree ->
+ fileTree.addToAntBuilder(ant, 'resources',
FileCollection.AntType.ResourceCollection)
}
- // The license rules below were manually copied from
lucene/common-build.xml, there is currently no mechanism to sync them
-
// BSD 4-clause stuff (is disallowed below)
substringMatcher(licenseFamilyCategory: "BSD4 ",
licenseFamilyName: "Original BSD License (with advertising clause)") {
pattern(substring: "All advertising materials")
@@ -160,7 +214,7 @@ class RatTask extends DefaultTask {
// ICU license
pattern(substring: "Permission is hereby granted, free of
charge, to any person obtaining a copy")
// ui-grid
- pattern(substring: " ; License: MIT")
+ pattern(substring: " ; License: MIT")
}
// Apache
@@ -186,8 +240,8 @@ class RatTask extends DefaultTask {
}
}
- def printUnknownFiles() {
- def ratXml = new XmlParser().parse(xmlReport)
+ def printUnknownFiles(File reportFile) {
+ def ratXml = new XmlParser().parse(reportFile)
def errors = []
ratXml.resource.each { resource ->
if (resource.'license-approval'.@name[0] == "false") {
@@ -201,14 +255,15 @@ class RatTask extends DefaultTask {
}
@TaskAction
- def rat() {
+ def execute() {
def origEncoding = System.getProperty("file.encoding")
try {
- generateXmlReport()
- printUnknownFiles()
+ File reportFile = xmlReport.get().asFile
+ generateReport(reportFile)
+ printUnknownFiles(reportFile)
} finally {
if (System.getProperty("file.encoding") != origEncoding) {
- throw new GradleException("Insane: rat changed file.encoding
to ${System.getProperty('file.encoding')}?")
+ throw new GradleException("Something is wrong: Apache RAT
changed file.encoding to ${System.getProperty('file.encoding')}?")
}
}
}
