jdaugherty commented on code in PR #16528: URL: https://github.com/apache/grails-core/pull/16528#discussion_r4190559320
########## build-logic/docs-core/src/main/groovy/grails/doc/gradle/ConvertAsciidocTask.groovy: ########## @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package grails.doc.gradle + +import javax.inject.Inject + +import groovy.transform.CompileStatic + +import org.gradle.api.DefaultTask +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.FileSystemOperations +import org.gradle.api.file.FileVisitDetails +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputDirectory +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction +import org.gradle.api.tasks.util.PatternSet +import org.gradle.workers.WorkQueue +import org.gradle.workers.WorkerExecutor + +/** + * Converts the AsciiDoc documents of a directory to HTML pages. + * + * <p>Every document matching {@link #getIncludes()} and not {@link #getExcludes()} - by default + * every {@code .adoc} file whose name does not start with an underscore - is converted on its + * own, with its own directory as the base directory, to the same relative path under + * {@link #getOutputDir()}. The {@code images} directory of the source directory is copied + * alongside the pages.</p> + * + * <p>Like the Asciidoctor Gradle plugin, the task sets {@code revnumber} and the + * {@code gradle-project-name}, {@code gradle-project-group} and {@code gradle-project-version} + * attributes from the project, and {@code includedir} to the source directory, unless + * {@link #getAttributes()} sets them.</p> + * + * <p>The conversion runs in a forked worker process, like {@link PublishGuideTask}, so the JRuby + * runtime AsciidoctorJ starts never lands in the Gradle daemon.</p> + * + * @since 8.0 + */ +@CompileStatic +@CacheableTask +abstract class ConvertAsciidocTask extends DefaultTask { + + @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) + abstract DirectoryProperty getSourceDir() + + @Input + abstract ListProperty<String> getIncludes() + + @Input + abstract ListProperty<String> getExcludes() + + /** + * The document attributes. Values are converted to strings before the documents are converted. + */ + @Input + abstract MapProperty<String, Object> getAttributes() + + /** The attributes describing the project, which {@link #getAttributes()} can override. */ + @Input + abstract MapProperty<String, String> getProjectAttributes() + + @OutputDirectory + abstract DirectoryProperty getOutputDir() + + /** The JVM arguments of the worker process the documents are converted in. */ + @Internal + abstract ListProperty<String> getJvmArgs() + + /** The maximum heap of the worker process the documents are converted in. */ + @Internal + abstract Property<String> getMaxHeapSize() + + @Inject + abstract WorkerExecutor getWorkerExecutor() + + @Inject + abstract FileSystemOperations getFileSystemOperations() + + @Inject + abstract ObjectFactory getObjects() + + ConvertAsciidocTask() { + group = 'documentation' + description = 'Converts the AsciiDoc documents to HTML.' + includes.convention(['**/*.adoc']) + excludes.convention(['**/_*']) Review Comment: Added in b1fef67: `ConvertAsciidocTaskSpec` runs the task in TestKit builds and covers the `_*` exclude, the `images` copy, `revnumber`, `gradle-project-*` and `includedir`, an attribute overriding a project attribute, included-file changes and build-cache relocation. `FetchTagsTaskSpec` covers the tag listing. ########## build-logic/vulnerability-scan/src/main/groovy/org/apache/grails/buildsrc/scan/OsvVulnerabilityService.groovy: ########## @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.buildsrc.scan + +import java.net.http.HttpRequest +import java.nio.charset.StandardCharsets +import java.nio.file.Path +import java.time.Duration +import java.util.concurrent.ConcurrentHashMap + +import groovy.json.JsonOutput +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic + +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.Property +import org.gradle.api.services.BuildService +import org.gradle.api.services.BuildServiceParameters + +/** + * Looks the vulnerabilities of Maven artifacts up in the <a href="https://osv.dev";>OSV</a> database. + * + * <p>Every answer is kept on disk for {@link Parameters#getCacheTtl() a while}, so builds that run again shortly + * after, in this or another checkout sharing the cache directory, do not ask the API again. Entries older than that + * are fetched again.</p> + * + * <p>The service is shared by the scan tasks of every project and asks the API one task at a time, which keeps the + * build well under the API's rate limits.</p> + */ +@CompileStatic +abstract class OsvVulnerabilityService implements BuildService<Parameters>, AutoCloseable { + + static final String NAME = 'OSV' + static final String DEFAULT_API_URL = 'https://api.osv.dev' + static final Duration DEFAULT_CACHE_TTL = Duration.ofHours(1) + + private static final int BATCH_SIZE = 1000 + + interface Parameters extends BuildServiceParameters { + + /** The OSV API, {@value OsvVulnerabilityService#DEFAULT_API_URL} unless a test points it elsewhere. */ + Property<String> getApiUrl() + + /** Where the answers of the API are kept between builds, shared with the other databases. */ + DirectoryProperty getCacheDirectory() + + /** How long an answer is used before the API is asked again. */ + Property<Duration> getCacheTtl() + } + + private final Map<String, List<String>> idsByCoordinate = new ConcurrentHashMap<>() + private final Map<String, Map<String, Object>> vulnerabilitiesById = new ConcurrentHashMap<>() + private final ApiClient client = new ApiClient('OSV API') + private ApiCache queryCache + private ApiCache vulnerabilityCache + + /** + * Returns the vulnerabilities known for each of the given {@code group:artifact:version} coordinates, as the + * OSV records of them. + */ + synchronized Map<String, List<Map<String, Object>>> vulnerabilities(Collection<String> coordinates) { + Collection<String> unique = coordinates.toUnique() + List<String> missing = unique.findAll { String coordinate -> idsFor(coordinate) == null }.toList() + if (missing) { + queryIds(missing) + } + + Map<String, List<Map<String, Object>>> vulnerabilities = [:] + for (String coordinate : unique) { + vulnerabilities.put(coordinate, idsByCoordinate.get(coordinate).collect { String id -> vulnerability(id) }) + } + vulnerabilities + } + + @Override + void close() { + client.close() + } + + private List<String> idsFor(String coordinate) { + List<String> ids = idsByCoordinate.get(coordinate) + if (ids == null) { + Map<String, Object> cached = queryCache().read(coordinate) + if (cached != null) { + ids = (List<String>) cached.get('ids') + idsByCoordinate.put(coordinate, ids) + } + } + ids + } + + private void queryIds(List<String> coordinates) { + coordinates.collate(BATCH_SIZE).each { List<String> batch -> + Map<String, List<String>> ids = batch.collectEntries { [(it): new ArrayList<String>()] } + Map<String, String> pageTokens = [:] + List<String> pending = batch + while (pending) { + List<Map<String, Object>> queries = pending.collect { String coordinate -> query(coordinate, pageTokens.get(coordinate)) } + List<Map<String, Object>> results = resultsOf(ApiClient.json(client.send(post('/v1/querybatch', [queries: queries])))) + pageTokens.clear() + pending.eachWithIndex { String coordinate, int index -> + Map<String, Object> result = results[index] ?: [:] Review Comment: Added in b1fef67, with a spec that answers a batch with fewer results than queries and checks that nothing was cached for the coordinates left out. ########## build-logic/vulnerability-scan/build.gradle: ########## @@ -16,26 +16,29 @@ * specific language governing permissions and limitations * under the License. */ + plugins { id 'groovy-gradle-plugin' } version = '0.0.1' group = 'org.apache.grails.buildsrc' -// The Sonatype scan plugin is a fat jar that bundles Guava, Groovy, Jackson and other libraries without -// relocating them. It is kept in its own project so that those bundled classes never reach the settings -// classpath that every build shares when it requests a plugin from ':build-logic'; there they would shadow -// the real dependencies of other settings plugins (for example the Guava the dependency-analysis plugin -// requires). -project.layout.projectDirectory.file('../../gradle.properties').asFile.withInputStream { - Properties props = new Properties() - props.load(it) - project.ext.gradleProperties = props +apply { + from project.layout.projectDirectory.file('../../dependencies.gradle') } dependencies { - implementation "org.sonatype.gradle.plugins:scan-gradle-plugin:${gradleProperties.sonatypeScanPluginVersion}" + testImplementation "org.spockframework:spock-core:${gradleBomDependencyVersions['gradle-spock.version']}" + testImplementation gradleTestKit() + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' +} + +tasks.named('test', Test) { Review Comment: Added in b1fef67: a `Build Logic` job in the CI workflow runs `./gradlew check` in `build-logic`. ########## build-logic/docs-core/src/main/groovy/grails/doc/git/FetchTagsTask.groovy: ########## @@ -27,52 +27,73 @@ import javax.inject.Inject import groovy.transform.CompileStatic +import org.gradle.api.DefaultTask import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty import org.gradle.api.file.RegularFileProperty import org.gradle.api.model.ObjectFactory import org.gradle.api.provider.Property import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.Exec import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Internal import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.TaskAction +import org.gradle.process.ExecOperations +import org.gradle.process.ExecSpec +/** + * Writes the tags of the repository, newest first, to {@link #getTagsFile()}; outside a git + * repository it writes {@link #getDefaultTag()} instead. + */ @CompileStatic @CacheableTask -abstract class FetchTagsTask extends Exec { +abstract class FetchTagsTask extends DefaultTask { @Input final Property<LocalDateTime> cacheDate // allows for forcing refreshing the tags, defaults to once a day @Input final Property<String> defaultTag // if no git repo present + @Input + final Property<Boolean> gitRepository + + @Internal + final DirectoryProperty repositoryDir + @OutputFile final RegularFileProperty tagsFile + @Inject + abstract ExecOperations getExecOperations() + @Inject FetchTagsTask(ObjectFactory objectFactory, Project project) { group = 'documentation' cacheDate = objectFactory.property(LocalDateTime).convention(LocalDate.now().atStartOfDay()) Review Comment: Fixed in b1fef67: the task is an `@UntrackedTask` now and the unused `cacheDate` is gone. `FetchTagsTaskSpec` adds a tag between two runs and checks that the reused entry lists it. ########## build-logic/vulnerability-scan/src/main/groovy/org/apache/grails/buildsrc/scan/ApiClient.groovy: ########## @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.buildsrc.scan + +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration + +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic + +import org.gradle.api.GradleException +import org.gradle.api.logging.Logger +import org.gradle.api.logging.Logging + +/** + * Sends the requests to a vulnerability database, retrying those that were throttled or failed on the way. + */ +@CompileStatic +class ApiClient implements AutoCloseable { + + private static final Logger LOG = Logging.getLogger(ApiClient) + private static final int MAX_ATTEMPTS = 4 + private static final int MAX_ERROR_BODY = 500 + + private final String name + private HttpClient httpClient + + /** + * @param name what the API is called in messages, such as {@code OSV API} + */ + ApiClient(String name) { + this.name = name + } + + /** + * Sends the request and returns the answer, once it is a {@code 200}. A request that was throttled, failed + * with a server error or did not get through is sent again, after the time the API asks for if it does. + * + * @param hints what to tell the user when the API answers with one of these statuses + */ + HttpResponse<String> send(HttpRequest request, Map<Integer, String> hints = [:]) { + IOException failure = null + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + HttpResponse<String> response = client().send(request, HttpResponse.BodyHandlers.ofString()) + int status = response.statusCode() + if (status == 200) { + return response + } + if (status != 429 && status < 500) { + String body = response.body() ?: '' + throw new GradleException("The ${name} answered ${request.method()} ${request.uri()} with ${status}" + + "${hints.get(status) ? " (${hints.get(status)})" : ''}: " + + "${body.length() > MAX_ERROR_BODY ? body.substring(0, MAX_ERROR_BODY) + '…' : body}") + } + LOG.info('The {} answered {} {} with {}, attempt {} of {}', name, request.method(), request.uri(), + status, attempt, MAX_ATTEMPTS) + if (attempt < MAX_ATTEMPTS) { + sleep(retryDelay(response, attempt)) + } + } + catch (IOException e) { + failure = e + LOG.info('Could not reach the {} at {}, attempt {} of {}', name, request.uri(), attempt, MAX_ATTEMPTS, e) + if (attempt < MAX_ATTEMPTS) { + sleep(Duration.ofSeconds(1L << attempt).toMillis()) + } + } + } + throw new GradleException("The ${name} did not answer ${request.method()} ${request.uri()} after " + Review Comment: Fixed in b1fef67: once the attempts run out, the message has the last status and body ("still answered ... with 429 after 4 attempts: ..."), and the wait uses `Thread.sleep`, turning an interrupt into a `BuildCancelledException`. A spec covers the throttled case. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
