This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
The following commit(s) were added to refs/heads/main by this push:
new 47febf5 Add `excludes` to `license resolve` config (#117)
47febf5 is described below
commit 47febf5b15c0c21c37a8ccaa40c1e96d7e8c430f
Author: kezhenxu94 <[email protected]>
AuthorDate: Tue Jun 14 16:11:57 2022 +0800
Add `excludes` to `license resolve` config (#117)
---
README.md | 8 ++++--
pkg/deps/config.go | 41 ++++++++++++++++++++++++++++++
pkg/deps/golang.go | 22 ++++++++--------
pkg/deps/maven.go | 73 ++++++++++++++++++++----------------------------------
pkg/deps/npm.go | 24 ++++++------------
pkg/deps/result.go | 8 ++++++
6 files changed, 100 insertions(+), 76 deletions(-)
diff --git a/README.md b/README.md
index 55d81d3..405d1d0 100644
--- a/README.md
+++ b/README.md
@@ -769,6 +769,9 @@ dependency: # <15>
version: dependency-version # <19>
license: Apache-2.0 # <20>
threshold: 75 # <21>
+ excludes: # <22>
+ - name: dependency-name # the same format as <18>
+ version: dependency-version # the same format as <19>
```
1. The `header` section is configurations for source codes license header.
@@ -788,10 +791,11 @@ dependency: # <15>
15. The `dependency` section is configurations for resolving dependencies'
licenses.
16. The `files` are the files that declare the dependencies of a project,
typically, `go.mod` in Go project, `pom.xml` in maven project, and
`package.json` in NodeJS project. If it's a relative path, it's relative to the
`.licenserc.yaml`.
17. Declare the licenses which cannot be identified by this tool.
-18. The `name` of the dependency, The name is different for different
projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project,
`PackageName` in NodeJS project.
-19. The `version` of the dependency, it's locked, preventing license changed
between different versions.
+18. The `name` of the dependency, The name is different for different
projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project,
`PackageName` in NodeJS project. You can use file pattern as described in [the
doc](https://pkg.go.dev/path/filepath#Match).
+19. The `version` of the dependency, comma seperated string (such as
`1.0,2.0,3.0`), if this is empty, it means all versions of the dependency.
20. The [SPDX ID](https://spdx.org/licenses/) of the dependency license.
21. The minimum percentage of the file that must contain license text for
identifying a license, default is `75`.
+22. The dependencies that should be excluded when analyzing the licenses, this
is useful when you declare the dependencies in `pom.xml` with `compile` scope
but don't distribute them in package. (Note that non-`compile` scope
dependencies are automatically excluded so you don't need to put them here).
**NOTE**: When the `SPDX-ID` is Apache-2.0 and the owner is Apache Software
foundation, the content would be [a dedicated
license](https://www.apache.org/legal/src-headers.html#headers) specified by
the ASF, otherwise, the license would be [the standard
one](https://www.apache.org/foundation/license-faq.html#Apply-My-Software).
diff --git a/pkg/deps/config.go b/pkg/deps/config.go
index d3a77ed..9993bad 100644
--- a/pkg/deps/config.go
+++ b/pkg/deps/config.go
@@ -20,6 +20,7 @@ package deps
import (
"os"
"path/filepath"
+ "strings"
)
// DefaultCoverageThreshold is the minimum percentage of the file
@@ -31,6 +32,7 @@ type ConfigDeps struct {
Threshold int `yaml:"threshold"`
Files []string `yaml:"files"`
Licenses []*ConfigDepLicense `yaml:"licenses"`
+ Excludes []Exclude `yaml:"excludes"`
}
type ConfigDepLicense struct {
@@ -39,6 +41,11 @@ type ConfigDepLicense struct {
License string `yaml:"license"`
}
+type Exclude struct {
+ Name string `yaml:"name"`
+ Version string `yaml:"version"`
+}
+
func (config *ConfigDeps) Finalize(configFile string) error {
configFileAbsPath, err := filepath.Abs(configFile)
if err != nil {
@@ -58,3 +65,37 @@ func (config *ConfigDeps) Finalize(configFile string) error {
return nil
}
+
+func (config *ConfigDeps) GetUserConfiguredLicense(name, version string)
(string, bool) {
+ for _, license := range config.Licenses {
+ if matched, _ := filepath.Match(license.Name, name); !matched
&& license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return license.License, true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return license.License, true
+ }
+ }
+ }
+ return "", false
+}
+
+func (config *ConfigDeps) IsExcluded(name, version string) bool {
+ for _, license := range config.Excludes {
+ if matched, _ := filepath.Match(license.Name, name); !matched
&& license.Name != name {
+ continue
+ }
+ if license.Version == "" {
+ return true
+ }
+ for _, v := range strings.Split(license.Version, ",") {
+ if v == version {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/pkg/deps/golang.go b/pkg/deps/golang.go
index 842be06..700329e 100644
--- a/pkg/deps/golang.go
+++ b/pkg/deps/golang.go
@@ -27,7 +27,6 @@ import (
"os/exec"
"path/filepath"
"regexp"
- "strings"
"github.com/apache/skywalking-eyes/internal/logger"
"github.com/apache/skywalking-eyes/pkg/license"
@@ -86,17 +85,16 @@ func (resolver *GoModResolver) Resolve(goModFile string,
config *ConfigDeps, rep
func (resolver *GoModResolver) ResolvePackages(modules []*packages.Module,
config *ConfigDeps, report *Report) error {
for _, module := range modules {
func() {
- for _, l := range config.Licenses {
- for _, version := range
strings.Split(l.Version, ",") {
- if l.Name == module.Path && version ==
module.Version {
- report.Resolve(&Result{
- Dependency:
module.Path,
- LicenseSpdxID:
l.License,
- Version:
module.Version,
- })
- return
- }
- }
+ if config.IsExcluded(module.Path, module.Version) {
+ return
+ }
+ if l, ok :=
config.GetUserConfiguredLicense(module.Path, module.Version); ok {
+ report.Resolve(&Result{
+ Dependency: module.Path,
+ LicenseSpdxID: l,
+ Version: module.Version,
+ })
+ return
}
err := resolver.ResolvePackageLicense(config, module,
report)
if err != nil {
diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go
index 28c5c08..7220823 100644
--- a/pkg/deps/maven.go
+++ b/pkg/deps/maven.go
@@ -58,14 +58,14 @@ func (resolver *MavenPomResolver) Resolve(mavenPomFile
string, config *ConfigDep
return err
}
- deps, err := resolver.LoadDependencies()
+ deps, err := resolver.LoadDependencies(config)
if err != nil {
// attempt to download dependencies
if err = resolver.DownloadDeps(); err != nil {
return fmt.Errorf("dependencies download error")
}
// load again
- deps, err = resolver.LoadDependencies()
+ deps, err = resolver.LoadDependencies(config)
if err != nil {
return err
}
@@ -125,7 +125,7 @@ func (resolver *MavenPomResolver) DownloadDeps() error {
return install.Run()
}
-func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) {
+func (resolver *MavenPomResolver) LoadDependencies(config *ConfigDeps)
([]*Dependency, error) {
buf := bytes.NewBuffer(nil)
cmd := exec.Command(resolver.maven, "dependency:tree") // #nosec G204
@@ -138,7 +138,7 @@ func (resolver *MavenPomResolver) LoadDependencies()
([]*Dependency, error) {
return nil, err
}
- deps := LoadDependencies(buf.Bytes())
+ deps := LoadDependencies(buf.Bytes(), config)
return deps, nil
}
@@ -146,24 +146,20 @@ func (resolver *MavenPomResolver) LoadDependencies()
([]*Dependency, error) {
func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency,
config *ConfigDeps, report *Report) error {
for _, dep := range deps {
func() {
- for _, l := range config.Licenses {
- for _, version := range
strings.Split(l.Version, ",") {
- if l.Name == fmt.Sprintf("%s:%s",
strings.Join(dep.GroupID, "."), dep.ArtifactID) && version == dep.Version {
- report.Resolve(&Result{
- Dependency:
dep.String(),
- LicenseSpdxID:
l.License,
- Version:
dep.Version,
- })
- return
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(dep.Name(),
dep.Version); ok {
+ report.Resolve(&Result{
+ Dependency: dep.Name(),
+ LicenseSpdxID: l,
+ Version: dep.Version,
+ })
+ return
}
state := NotFound
err := resolver.ResolveLicense(config, &state, dep,
report)
if err != nil {
- logger.Log.Warnf("Failed to resolve the license
of <%s>: %v\n", dep.String(), state.String())
+ logger.Log.Warnf("Failed to resolve the license
of <%s>: %v\n", dep.Name(), state.String())
report.Skip(&Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseSpdxID: Unknown,
Version: dep.Version,
})
@@ -177,7 +173,7 @@ func (resolver *MavenPomResolver) ResolveDependencies(deps
[]*Dependency, config
func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state
*State, dep *Dependency, report *Report) error {
result1, err1 := resolver.ResolveJar(config, state,
filepath.Join(resolver.repo, dep.Path(), dep.Jar()), dep.Version)
if result1 != nil {
- result1.Dependency = dep.String()
+ result1.Dependency = dep.Name()
report.Resolve(result1)
return nil
}
@@ -188,7 +184,7 @@ func (resolver *MavenPomResolver) ResolveLicense(config
*ConfigDeps, state *Stat
return nil
}
- return fmt.Errorf("failed to resolve license for <%s> from jar or pom:
%+v, %+v", dep.String(), err1, err2)
+ return fmt.Errorf("failed to resolve license for <%s> from jar or pom:
%+v, %+v", dep.Name(), err1, err2)
}
// ResolveLicenseFromPom search for license in the pom file, which may appear
in the header comments or in license element of xml
@@ -202,7 +198,7 @@ func (resolver *MavenPomResolver)
ResolveLicenseFromPom(config *ConfigDeps, stat
if pom != nil && len(pom.Licenses) != 0 {
return &Result{
- Dependency: dep.String(),
+ Dependency: dep.Name(),
LicenseFilePath: pomFile,
LicenseContent: pom.Raw(),
LicenseSpdxID: pom.AllLicenses(config),
@@ -215,7 +211,7 @@ func (resolver *MavenPomResolver)
ResolveLicenseFromPom(config *ConfigDeps, stat
return nil, err
} else if headerComments != "" {
*state |= FoundLicenseInPomHeader
- return resolver.IdentifyLicense(config, pomFile, dep.String(),
headerComments, dep.Version)
+ return resolver.IdentifyLicense(config, pomFile, dep.Name(),
headerComments, dep.Version)
}
return nil, fmt.Errorf("not found in pom file")
@@ -287,7 +283,7 @@ func SeemLicense(content string) bool {
return reMaybeLicense.MatchString(content)
}
-func LoadDependencies(data []byte) []*Dependency {
+func LoadDependencies(data []byte, config *ConfigDeps) []*Dependency {
depsTree := LoadDependenciesTree(data)
cnt := 0
@@ -299,6 +295,9 @@ func LoadDependencies(data []byte) []*Dependency {
queue := []*Dependency{}
for _, depTree := range depsTree {
+ if config.IsExcluded(depTree.Name(), depTree.Version) {
+ continue
+ }
queue = append(queue, depTree)
for len(queue) > 0 {
dep := queue[0]
@@ -326,9 +325,8 @@ func LoadDependenciesTree(data []byte) []*Dependency {
deps := make([]*Dependency, 0, len(rawDeps))
for _, rawDep := range rawDeps {
- gid := strings.Split(string(rawDep[reFind.SubexpIndex("gid")]),
".")
dep := &Dependency{
- GroupID: gid,
+ GroupID: string(rawDep[reFind.SubexpIndex("gid")]),
ArtifactID: string(rawDep[reFind.SubexpIndex("aid")]),
Packaging:
string(rawDep[reFind.SubexpIndex("packaging")]),
Version:
string(rawDep[reFind.SubexpIndex("version")]),
@@ -406,9 +404,8 @@ func (s *State) String() string {
}
type Dependency struct {
- GroupID []string
- ArtifactID, Version, Packaging, Scope string
- TransitiveDeps []*Dependency
+ GroupID, ArtifactID, Version, Packaging, Scope string
+ TransitiveDeps []*Dependency
}
func (dep *Dependency) Clone() *Dependency {
@@ -430,7 +427,7 @@ func (dep *Dependency) Count() int {
}
func (dep *Dependency) Path() string {
- return fmt.Sprintf("%v/%v/%v", strings.Join(dep.GroupID, "/"),
dep.ArtifactID, dep.Version)
+ return fmt.Sprintf("%v/%v/%v", strings.ReplaceAll(dep.GroupID, ".",
"/"), dep.ArtifactID, dep.Version)
}
func (dep *Dependency) Pom() string {
@@ -441,24 +438,8 @@ func (dep *Dependency) Jar() string {
return fmt.Sprintf("%v-%v.jar", dep.ArtifactID, dep.Version)
}
-func (dep *Dependency) String() string {
- buf := bytes.NewBuffer(nil)
- w := bufio.NewWriter(buf)
-
- _, err := w.WriteString(fmt.Sprintf("%v:%v", strings.Join(dep.GroupID,
"."), dep.ArtifactID))
- if err != nil {
- logger.Log.Error(err)
- }
-
- for _, tDep := range dep.TransitiveDeps {
- _, err = w.WriteString(fmt.Sprintf("\n\t%v", tDep))
- if err != nil {
- logger.Log.Error(err)
- }
- }
-
- _ = w.Flush()
- return buf.String()
+func (dep *Dependency) Name() string {
+ return fmt.Sprintf("%v:%v", dep.GroupID, dep.ArtifactID)
}
// PomFile is used to extract license from the pom.xml file
diff --git a/pkg/deps/npm.go b/pkg/deps/npm.go
index 4cdb28f..c1d97dd 100644
--- a/pkg/deps/npm.go
+++ b/pkg/deps/npm.go
@@ -190,7 +190,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName,
pkgPath string, conf
Dependency: pkgName,
}
// resolve from the package.json file
- if err := resolver.ResolvePkgFile(result, pkgPath, config.Licenses);
err != nil {
+ if err := resolver.ResolvePkgFile(result, pkgPath, config); err != nil {
result.ResolveErrors = append(result.ResolveErrors, err)
}
@@ -203,7 +203,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName,
pkgPath string, conf
}
// ResolvePkgFile tries to find and parse the package.json file to capture the
license field
-func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string,
licenses []*ConfigDepLicense) error {
+func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string,
config *ConfigDeps) error {
expectedPkgFile := filepath.Join(pkgPath, PkgFileName)
packageInfo, err := resolver.ParsePkgFile(expectedPkgFile)
if err != nil {
@@ -211,13 +211,9 @@ func (resolver *NpmResolver) ResolvePkgFile(result
*Result, pkgPath string, lice
}
result.Version = packageInfo.Version
- for _, l := range licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == packageInfo.Name && version ==
packageInfo.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(packageInfo.Name,
packageInfo.Version); ok {
+ result.LicenseSpdxID = l
+ return nil
}
if lcs, ok := resolver.ResolveLicenseField(packageInfo.License); ok {
@@ -287,13 +283,9 @@ func (resolver *NpmResolver) ResolveLcsFile(result
*Result, pkgPath string, conf
if result.LicenseSpdxID != "" {
return nil
}
- for _, l := range config.Licenses {
- for _, version := range strings.Split(l.Version, ",") {
- if l.Name == info.Name() && version ==
result.Version {
- result.LicenseSpdxID = l.License
- return nil
- }
- }
+ if l, ok := config.GetUserConfiguredLicense(info.Name(),
result.Version); ok {
+ result.LicenseSpdxID = l
+ return nil
}
identifier, err := license.Identify(string(content),
config.Threshold)
if err != nil {
diff --git a/pkg/deps/result.go b/pkg/deps/result.go
index 95a318b..66f87df 100644
--- a/pkg/deps/result.go
+++ b/pkg/deps/result.go
@@ -20,6 +20,7 @@ package deps
import (
"fmt"
"math"
+ "sort"
"strings"
)
@@ -56,6 +57,13 @@ func (report *Report) Skip(result *Result) {
}
func (report *Report) String() string {
+ sort.SliceStable(report.Resolved, func(i, j int) bool {
+ return report.Resolved[i].Dependency <
report.Resolved[j].Dependency
+ })
+ sort.SliceStable(report.Skipped, func(i, j int) bool {
+ return report.Skipped[i].Dependency <
report.Skipped[j].Dependency
+ })
+
dWidth, lWidth, vWidth := .0, .0, .0
for _, r := range report.Skipped {
dWidth = math.Max(float64(len(r.Dependency)), dWidth)
