This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d7e54cf14441d895ef4fd26e228d9857a2439d2e Author: Michael Blow <michael.b...@couchbase.com> AuthorDate: Sat Nov 19 18:59:28 2022 -0500 [NO ISSUE] Process shadowed projects w/ license plugin Change-Id: Id21dcd3dc61bd4766e71c5d509e7ef6802e423be Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17290 Reviewed-by: Michael Blow <mb...@apache.org> Reviewed-by: Murtadha Hubail <mhub...@apache.org> Tested-by: Michael Blow <mb...@apache.org> Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> --- .../main/licenses/templates/asterix-license.ftl | 3 + .../src/main/licenses/templates/asterix-notice.ftl | 3 + .../src/main/licenses/templates/hyracks-notice.ftl | 3 + .../maven/license/DownloadLicensesMojo.java | 2 +- .../hyracks/maven/license/GenerateFileMojo.java | 120 ++++++++++++++++++--- .../apache/hyracks/maven/license/LicenseMojo.java | 47 +++++--- .../hyracks/maven/license/project/Project.java | 12 ++- 7 files changed, 159 insertions(+), 31 deletions(-) diff --git a/asterixdb/src/main/licenses/templates/asterix-license.ftl b/asterixdb/src/main/licenses/templates/asterix-license.ftl index 381d400a90..434ce447f6 100644 --- a/asterixdb/src/main/licenses/templates/asterix-license.ftl +++ b/asterixdb/src/main/licenses/templates/asterix-license.ftl @@ -44,6 +44,9 @@ ${license.content} <#assign isare = "are"/> </#if> <#list projects as p> + <#if p.shadowed> + <#continue/> + </#if> * ${p.name} (${p.groupId}:${p.artifactId}:${p.version}) <#list p.locations as loc> - ${loc}${p.jarName} diff --git a/asterixdb/src/main/licenses/templates/asterix-notice.ftl b/asterixdb/src/main/licenses/templates/asterix-notice.ftl index c825397c1a..5090cb0868 100644 --- a/asterixdb/src/main/licenses/templates/asterix-notice.ftl +++ b/asterixdb/src/main/licenses/templates/asterix-notice.ftl @@ -34,6 +34,9 @@ AsterixDB utilizes many libraries, which come with the following applicable NOTI <#assign projects = e.getValue()/> <#list projects as p> + <#if p.shadowed> + <#continue/> + </#if> ${p.name} (${p.groupId}:${p.artifactId}:${p.version}) <#list p.locations as loc> - ${loc}${p.jarName} diff --git a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl index 8a2eb05d29..0cf9378659 100644 --- a/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl +++ b/hyracks-fullstack/hyracks-fullstack-license/src/main/licenses/templates/hyracks-notice.ftl @@ -30,6 +30,9 @@ Hyracks and Algebricks utilize many libraries, which come with the following app <#assign noticeText = e.getKey()/> <#assign projects = e.getValue()/> <#list projects as p> + <#if p.shadowed> + <#continue/> + </#if> ${p.name} (${p.groupId}:${p.artifactId}:${p.version}) <#list p.locations as loc> - ${loc}${p.jarName} diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java index 293c08c6cf..c8980cd3da 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/DownloadLicensesMojo.java @@ -63,7 +63,7 @@ public class DownloadLicensesMojo extends LicenseMojo { String fileName = entry.getLicense().getContentFile(false); doDownload(timeoutMillis, i, url, fileName); }); - } catch (ProjectBuildingException e) { + } catch (ProjectBuildingException | IOException e) { throw new MojoExecutionException("Unexpected exception: " + e, e); } } diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java index e69887c289..b3a0bb299c 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java @@ -39,6 +39,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -52,15 +53,22 @@ import java.util.jar.JarFile; import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hyracks.maven.license.freemarker.IndentDirective; import org.apache.hyracks.maven.license.freemarker.LoadFileDirective; import org.apache.hyracks.maven.license.project.LicensedProjects; import org.apache.hyracks.maven.license.project.Project; +import org.apache.maven.artifact.Artifact; +import org.apache.maven.artifact.DefaultArtifact; +import org.apache.maven.artifact.handler.ArtifactHandler; +import org.apache.maven.artifact.handler.DefaultArtifactHandler; import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; import org.apache.maven.plugins.annotations.ResolutionScope; +import org.apache.maven.project.MavenProject; import org.apache.maven.project.ProjectBuildingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -106,6 +114,9 @@ public class GenerateFileMojo extends LicenseMojo { @Parameter private boolean stripFoundationAssertionFromNotices = false; + @Parameter + private boolean validateShadowLicenses = false; + private SortedMap<String, SortedSet<Project>> noticeMap; @java.lang.Override @@ -139,11 +150,12 @@ public class GenerateFileMojo extends LicenseMojo { } licenseSpecs.addAll(urlToLicenseMap.values()); for (LicenseSpec license : licenseSpecs) { - resolveArtifactContent(license, true); + resolveArtifactContent(license, true, false); } } - private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort) throws IOException { + private String resolveArtifactContent(ArtifactSpec artifact, boolean bestEffort, boolean suppressWarning) + throws IOException { if (artifact.getContent() == null) { getLog().debug("Resolving content for " + artifact.getUrl() + " (" + artifact.getContentFile() + ")"); File cFile = new File(artifact.getContentFile()); @@ -157,7 +169,9 @@ public class GenerateFileMojo extends LicenseMojo { } if (!cFile.exists()) { if (!bestEffort) { - getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl()); + if (!suppressWarning) { + getLog().warn("MISSING: content file (" + cFile + ") for url: " + artifact.getUrl()); + } artifact.setContent("MISSING: " + artifact.getContentFile() + " (" + artifact.getUrl() + ")"); } } else { @@ -264,10 +278,12 @@ public class GenerateFileMojo extends LicenseMojo { for (Project p : lps.getProjects()) { String licenseText = p.getLicenseText(); if (licenseText == null) { - warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE, - "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense() - + ")"); - licenseText = resolveArtifactContent(lps.getLicense(), false); + if (validateProjectLicense(p)) { + warnUnlessFlag(p.gav(), IGNORE_MISSING_EMBEDDED_LICENSE, + "Using license other than from within artifact: " + p.gav() + " (" + lps.getLicense() + + ")"); + } + licenseText = resolveArtifactContent(lps.getLicense(), false, !validateProjectLicense(p)); } LicenseSpec spec = lps.getLicense(); if (spec.getDisplayName() == null) { @@ -293,6 +309,10 @@ public class GenerateFileMojo extends LicenseMojo { licenseMap = licenseMap2; } + private boolean validateProjectLicense(Project p) { + return !p.isShadowed() || validateShadowLicenses; + } + private Set<Project> getProjects() { Set<Project> projects = new HashSet<>(); licenseMap.values().forEach(p -> projects.addAll(p.getProjects())); @@ -305,9 +325,11 @@ public class GenerateFileMojo extends LicenseMojo { String noticeText = p.getNoticeText(); if (noticeText == null && noticeOverrides.containsKey(p.gav())) { String noticeUrl = noticeOverrides.get(p.gav()); - warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE, - "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")"); - p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false)); + if (validateProjectLicense(p)) { + warnUnlessFlag(p.gav(), IGNORE_NOTICE_OVERRIDE, + "Using notice other than from within artifact: " + p.gav() + " (" + noticeUrl + ")"); + } + p.setNoticeText(resolveArtifactContent(new NoticeSpec(noticeUrl), false, p.isShadowed())); } else if (noticeText == null && !noticeOverrides.containsKey(p.gav()) && Boolean.TRUE.equals(getProjectFlag(p.gav(), IGNORE_NOTICE_OVERRIDE))) { getLog().warn(p + " has IGNORE_NOTICE_OVERRIDE flag set, but no override defined..."); @@ -338,20 +360,20 @@ public class GenerateFileMojo extends LicenseMojo { // TODO(mblow): this will match *any* NOTICE[.txt] file located within the artifact- this seems way too liberal resolveArtifactFiles("NOTICE", IGNORE_MISSING_EMBEDDED_NOTICE, ALTERNATE_NOTICE_FILE, entry -> entry.getName().matches("(.*/|^)" + "NOTICE" + "(.txt)?"), Project::setNoticeText, - text -> stripFoundationAssertionFromNotices ? FOUNDATION_PATTERN.matcher(text).replaceAll("") : text); + getNoticeFileContentTransformer(), !validateShadowLicenses); } private void resolveLicenseFiles() throws MojoExecutionException, IOException { // TODO(mblow): this will match *any* LICENSE[.txt] file located within the artifact- this seems way too liberal resolveArtifactFiles("LICENSE", IGNORE_MISSING_EMBEDDED_LICENSE, ALTERNATE_LICENSE_FILE, entry -> entry.getName().matches("(.*/|^)" + "LICENSE" + "(.txt)?"), Project::setLicenseText, - UnaryOperator.identity()); + UnaryOperator.identity(), !validateShadowLicenses); } private void resolveArtifactFiles(final String name, final ProjectFlag ignoreFlag, final ProjectFlag alternateFilenameFlag, final Predicate<JarEntry> filter, - final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer) - throws MojoExecutionException, IOException { + final BiConsumer<Project, String> consumer, final UnaryOperator<String> contentTransformer, + boolean skipShadowed) throws MojoExecutionException, IOException { for (Project p : getProjects()) { File artifactFile = new File(p.getArtifactPath()); if (!artifactFile.exists()) { @@ -359,6 +381,9 @@ public class GenerateFileMojo extends LicenseMojo { } else if (!artifactFile.getName().endsWith(".jar")) { getLog().info("Skipping unknown artifact file type: " + artifactFile); continue; + } else if (skipShadowed && p.isShadowed()) { + getLog().info("Skipping shadowed project: " + p.gav()); + continue; } String alternateFilename = (String) getProjectFlag(p.gav(), alternateFilenameFlag); Predicate<JarEntry> finalFilter = @@ -404,4 +429,71 @@ public class GenerateFileMojo extends LicenseMojo { } return matches; } + + private UnaryOperator<String> getNoticeFileContentTransformer() { + UnaryOperator<String> transformer; + if (stripFoundationAssertionFromNotices) { + transformer = text -> FOUNDATION_PATTERN.matcher(text).replaceAll(""); + } else { + transformer = UnaryOperator.identity(); + } + return transformer; + } + + @java.lang.Override + protected void gatherProjectDependencies(MavenProject project, + Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, + Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException { + super.gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap); + gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap); + } + + @java.lang.Override + protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, + Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException { + super.processExtraDependencies(dependencyLicenseMap, dependencyGavMap); + gatherShadowedDependencies(dependencyLicenseMap, dependencyGavMap); + } + + private void gatherShadowedDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, + Map<String, MavenProject> dependencyGavMap) throws MojoExecutionException, ProjectBuildingException { + Set<MavenProject> projects = new TreeSet<>(Comparator.comparing(MavenProject::getId)); + projects.addAll(dependencyLicenseMap.keySet()); + for (MavenProject p : projects) { + boolean finished = false; + File artifactFile = p.getArtifact().getFile(); + if (!artifactFile.exists()) { + throw new MojoExecutionException("Artifact file " + artifactFile + " does not exist!"); + } else if (!artifactFile.getName().endsWith(".jar")) { + getLog().info("Skipping unknown artifact file type: " + artifactFile); + finished = true; + } + if (!finished) { + try (JarFile jarFile = new JarFile(artifactFile)) { + SortedMap<String, JarEntry> matches = gatherMatchingEntries(jarFile, + entry -> entry.getName().matches("(.*/|^)" + "pom\\.properties")); + if (!matches.isEmpty()) { + for (JarEntry entry : matches.values()) { + Properties props = new Properties(); + props.load(jarFile.getInputStream(entry)); + String groupId = props.getProperty("groupId"); + String artifactId = props.getProperty("artifactId"); + String version = props.getProperty("version"); + String gav = groupId + ":" + artifactId + ":" + version; + if (!dependencyGavMap.containsKey(gav)) { + getLog().info("adding " + gav + " (shadowed from " + p.getId() + ")"); + ArtifactHandler handler = new DefaultArtifactHandler("jar"); + String[] gavParts = StringUtils.split(gav, ':'); + Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], + Artifact.SCOPE_COMPILE, "jar", null, handler); + processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, true); + } + } + } + } catch (IOException e) { + throw new MojoExecutionException(e); + } + } + } + } } diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java index 2206621fb7..73cc69559b 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseMojo.java @@ -70,6 +70,7 @@ import org.apache.maven.project.inheritance.ModelInheritanceAssembler; public abstract class LicenseMojo extends AbstractMojo { + private static final String SHADOWED_KEY = LicenseMojo.class.getName() + "_shadowed"; @Parameter protected List<Override> overrides = new ArrayList<>(); @@ -249,7 +250,7 @@ public abstract class LicenseMojo extends AbstractMojo { }); } - protected void addDependenciesToLicenseMap() throws ProjectBuildingException { + protected void addDependenciesToLicenseMap() throws ProjectBuildingException, MojoExecutionException, IOException { Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = gatherDependencies(); dependencyLicenseMap.forEach((depProject, value) -> { Set<String> locations = dependencySets.isEmpty() ? Collections.singleton(location) @@ -310,7 +311,9 @@ public abstract class LicenseMojo extends AbstractMojo { licenseUrl = fakeLicenseUrl; } } - addProject(new Project(depProject, depLocation, depProject.getArtifact().getFile()), + addProject( + new Project(depProject, depLocation, depProject.getArtifact().getFile(), + Boolean.parseBoolean(String.valueOf(depProject.getContextValue(SHADOWED_KEY)))), new LicenseSpec(licenseUrl, displayName), true); } @@ -363,11 +366,13 @@ public abstract class LicenseMojo extends AbstractMojo { } } - protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies() throws ProjectBuildingException { + protected Map<MavenProject, List<Pair<String, String>>> gatherDependencies() + throws ProjectBuildingException, MojoExecutionException, IOException { Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap = new HashMap<>(); Map<String, MavenProject> dependencyGavMap = new HashMap<>(); gatherProjectDependencies(project, dependencyLicenseMap, dependencyGavMap); + processExtraDependencies(dependencyLicenseMap, dependencyGavMap); for (Override override : overrides) { // Collect both <gav></gav> and <gavs><gav></gav><gav></gav>...</gavs> @@ -399,27 +404,40 @@ public abstract class LicenseMojo extends AbstractMojo { return dependencyLicenseMap; } - private void gatherProjectDependencies(MavenProject project, + protected void gatherProjectDependencies(MavenProject project, Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, - Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException { + Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException { final Set<Artifact> dependencyArtifacts = project.getArtifacts(); if (dependencyArtifacts != null) { for (Artifact depArtifact : dependencyArtifacts) { - processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap); + processArtifact(depArtifact, dependencyLicenseMap, dependencyGavMap, false); } } - for (String gav : extraDependencies) { - ArtifactHandler handler = new DefaultArtifactHandler("jar"); - String[] gavParts = StringUtils.split(gav, ':'); - Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE, - "jar", null, handler); - processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap); + } + + protected void processExtraDependencies(Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, + Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException, MojoExecutionException { + for (String extraDependency : extraDependencies) { + String[] gavParts = StringUtils.split(extraDependency, ':'); + String gav = gavParts[0] + ":" + gavParts[1] + ":" + gavParts[2]; + if (!dependencyGavMap.containsKey(gav)) { + ArtifactHandler handler = new DefaultArtifactHandler("jar"); + Artifact manualDep = new DefaultArtifact(gavParts[0], gavParts[1], gavParts[2], Artifact.SCOPE_COMPILE, + "jar", null, handler); + boolean shadowed = false; + if (gavParts.length > 3) { + shadowed = "shadowed".equals(gavParts[3]); + } + processArtifact(manualDep, dependencyLicenseMap, dependencyGavMap, shadowed); + } else { + getLog().warn("not adding extra dependency " + gav + " as it is already added"); + } } } - private void processArtifact(Artifact depArtifact, + protected void processArtifact(Artifact depArtifact, Map<MavenProject, List<Pair<String, String>>> dependencyLicenseMap, - Map<String, MavenProject> dependencyGavMap) throws ProjectBuildingException { + Map<String, MavenProject> dependencyGavMap, boolean shadowed) throws ProjectBuildingException { if (!excludedScopes.contains(depArtifact.getScope())) { MavenProject dep = resolveDependency(depArtifact); if (!depArtifact.isResolved()) { @@ -442,6 +460,7 @@ public abstract class LicenseMojo extends AbstractMojo { : (license1.getName() != null ? license1.getName() : "LICENSE_EMPTY_NAME_URL"); licenseUrls.add(new ImmutablePair<>(url, license1.getName())); } + dep.setContextValue(SHADOWED_KEY, shadowed); dependencyLicenseMap.put(dep, licenseUrls); } } diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java index e44914d7d4..a755c05ab5 100644 --- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java +++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/project/Project.java @@ -41,6 +41,7 @@ public class Project { private String licenseText; private String sourcePointer; private String classifier; + private final boolean shadowed; @JsonIgnore private MavenProject mavenProject; @@ -48,7 +49,7 @@ public class Project { public static final Comparator<Project> PROJECT_COMPARATOR = (o1, o2) -> o1.compareToken().compareTo(o2.compareToken()); - public Project(MavenProject project, String location, File artifactPath) { + public Project(MavenProject project, String location, File artifactPath, boolean shadowed) { mavenProject = project; name = project.getName(); groupId = project.getGroupId(); @@ -57,6 +58,7 @@ public class Project { url = project.getUrl(); classifier = project.getArtifact().getClassifier(); this.artifactPath = artifactPath.getPath(); + this.shadowed = shadowed; setLocation(location); } @@ -65,7 +67,8 @@ public class Project { @JsonProperty("artifactId") String artifactId, @JsonProperty("url") String url, @JsonProperty("version") String version, @JsonProperty("location") String location, @JsonProperty("artifactPath") String artifactPath, @JsonProperty("noticeText") String noticeText, - @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier) { + @JsonProperty("licenseText") String licenseText, @JsonProperty("classifier") String classifier, + @JsonProperty("shadowed") boolean shadowed) { this.name = name; this.groupId = groupId; this.artifactId = artifactId; @@ -76,6 +79,7 @@ public class Project { this.noticeText = noticeText; this.licenseText = licenseText; this.classifier = classifier; + this.shadowed = shadowed; } public String getName() { @@ -185,6 +189,10 @@ public class Project { this.sourcePointer = sourcePointer; } + public boolean isShadowed() { + return shadowed; + } + @Override public String toString() { return "Project [" + gav() + "]";