This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch config/excludes in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
commit 141b781379a2eefb57e76218a08d609727ab42de Author: kezhenxu94 <[email protected]> AuthorDate: Tue Jun 14 15:58:55 2022 +0800 Add `excludes` to `license resolve` config --- README.md | 8 ++++-- pkg/deps/config.go | 41 ++++++++++++++++++++++++++++++ pkg/deps/golang.go | 22 ++++++++-------- pkg/deps/maven.go | 73 ++++++++++++++++++++---------------------------------- pkg/deps/npm.go | 24 ++++++------------ pkg/deps/result.go | 8 ++++++ 6 files changed, 100 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 55d81d3..405d1d0 100644 --- a/README.md +++ b/README.md @@ -769,6 +769,9 @@ dependency: # <15> version: dependency-version # <19> license: Apache-2.0 # <20> threshold: 75 # <21> + excludes: # <22> + - name: dependency-name # the same format as <18> + version: dependency-version # the same format as <19> ``` 1. The `header` section is configurations for source codes license header. @@ -788,10 +791,11 @@ dependency: # <15> 15. The `dependency` section is configurations for resolving dependencies' licenses. 16. The `files` are the files that declare the dependencies of a project, typically, `go.mod` in Go project, `pom.xml` in maven project, and `package.json` in NodeJS project. If it's a relative path, it's relative to the `.licenserc.yaml`. 17. Declare the licenses which cannot be identified by this tool. -18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project. -19. The `version` of the dependency, it's locked, preventing license changed between different versions. +18. The `name` of the dependency, The name is different for different projects, `PackagePath` in Go project, `GroupID:ArtifactID` in maven project, `PackageName` in NodeJS project. You can use file pattern as described in [the doc](https://pkg.go.dev/path/filepath#Match). +19. The `version` of the dependency, comma seperated string (such as `1.0,2.0,3.0`), if this is empty, it means all versions of the dependency. 20. The [SPDX ID](https://spdx.org/licenses/) of the dependency license. 21. The minimum percentage of the file that must contain license text for identifying a license, default is `75`. +22. The dependencies that should be excluded when analyzing the licenses, this is useful when you declare the dependencies in `pom.xml` with `compile` scope but don't distribute them in package. (Note that non-`compile` scope dependencies are automatically excluded so you don't need to put them here). **NOTE**: When the `SPDX-ID` is Apache-2.0 and the owner is Apache Software foundation, the content would be [a dedicated license](https://www.apache.org/legal/src-headers.html#headers) specified by the ASF, otherwise, the license would be [the standard one](https://www.apache.org/foundation/license-faq.html#Apply-My-Software). diff --git a/pkg/deps/config.go b/pkg/deps/config.go index d3a77ed..c29d7de 100644 --- a/pkg/deps/config.go +++ b/pkg/deps/config.go @@ -20,6 +20,7 @@ package deps import ( "os" "path/filepath" + "strings" ) // DefaultCoverageThreshold is the minimum percentage of the file @@ -31,6 +32,7 @@ type ConfigDeps struct { Threshold int `yaml:"threshold"` Files []string `yaml:"files"` Licenses []*ConfigDepLicense `yaml:"licenses"` + Excludes []Exclude `yaml:"excludes"` } type ConfigDepLicense struct { @@ -39,6 +41,11 @@ type ConfigDepLicense struct { License string `yaml:"license"` } +type Exclude struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} + func (config *ConfigDeps) Finalize(configFile string) error { configFileAbsPath, err := filepath.Abs(configFile) if err != nil { @@ -58,3 +65,37 @@ func (config *ConfigDeps) Finalize(configFile string) error { return nil } + +func (conf *ConfigDeps) GetUserConfiguredLicense(name, version string) (string, bool) { + for _, license := range conf.Licenses { + if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name { + continue + } + if license.Version == "" { + return license.License, true + } + for _, v := range strings.Split(license.Version, ",") { + if v == version { + return license.License, true + } + } + } + return "", false +} + +func (conf *ConfigDeps) IsExcluded(name, version string) bool { + for _, license := range conf.Excludes { + if matched, _ := filepath.Match(license.Name, name); !matched && license.Name != name { + continue + } + if license.Version == "" { + return true + } + for _, v := range strings.Split(license.Version, ",") { + if v == version { + return true + } + } + } + return false +} diff --git a/pkg/deps/golang.go b/pkg/deps/golang.go index 842be06..9b97f3f 100644 --- a/pkg/deps/golang.go +++ b/pkg/deps/golang.go @@ -27,7 +27,6 @@ import ( "os/exec" "path/filepath" "regexp" - "strings" "github.com/apache/skywalking-eyes/internal/logger" "github.com/apache/skywalking-eyes/pkg/license" @@ -86,17 +85,16 @@ func (resolver *GoModResolver) Resolve(goModFile string, config *ConfigDeps, rep func (resolver *GoModResolver) ResolvePackages(modules []*packages.Module, config *ConfigDeps, report *Report) error { for _, module := range modules { func() { - for _, l := range config.Licenses { - for _, version := range strings.Split(l.Version, ",") { - if l.Name == module.Path && version == module.Version { - report.Resolve(&Result{ - Dependency: module.Path, - LicenseSpdxID: l.License, - Version: module.Version, - }) - return - } - } + if config.IsExcluded(module.Path, module.Version) { + return + } + if license, ok := config.GetUserConfiguredLicense(module.Path, module.Version); ok { + report.Resolve(&Result{ + Dependency: module.Path, + LicenseSpdxID: license, + Version: module.Version, + }) + return } err := resolver.ResolvePackageLicense(config, module, report) if err != nil { diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go index 28c5c08..53b353d 100644 --- a/pkg/deps/maven.go +++ b/pkg/deps/maven.go @@ -58,14 +58,14 @@ func (resolver *MavenPomResolver) Resolve(mavenPomFile string, config *ConfigDep return err } - deps, err := resolver.LoadDependencies() + deps, err := resolver.LoadDependencies(config) if err != nil { // attempt to download dependencies if err = resolver.DownloadDeps(); err != nil { return fmt.Errorf("dependencies download error") } // load again - deps, err = resolver.LoadDependencies() + deps, err = resolver.LoadDependencies(config) if err != nil { return err } @@ -125,7 +125,7 @@ func (resolver *MavenPomResolver) DownloadDeps() error { return install.Run() } -func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) { +func (resolver *MavenPomResolver) LoadDependencies(config *ConfigDeps) ([]*Dependency, error) { buf := bytes.NewBuffer(nil) cmd := exec.Command(resolver.maven, "dependency:tree") // #nosec G204 @@ -138,7 +138,7 @@ func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) { return nil, err } - deps := LoadDependencies(buf.Bytes()) + deps := LoadDependencies(buf.Bytes(), config) return deps, nil } @@ -146,24 +146,20 @@ func (resolver *MavenPomResolver) LoadDependencies() ([]*Dependency, error) { func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency, config *ConfigDeps, report *Report) error { for _, dep := range deps { func() { - for _, l := range config.Licenses { - for _, version := range strings.Split(l.Version, ",") { - if l.Name == fmt.Sprintf("%s:%s", strings.Join(dep.GroupID, "."), dep.ArtifactID) && version == dep.Version { - report.Resolve(&Result{ - Dependency: dep.String(), - LicenseSpdxID: l.License, - Version: dep.Version, - }) - return - } - } + if license, ok := config.GetUserConfiguredLicense(dep.Name(), dep.Version); ok { + report.Resolve(&Result{ + Dependency: dep.Name(), + LicenseSpdxID: license, + Version: dep.Version, + }) + return } state := NotFound err := resolver.ResolveLicense(config, &state, dep, report) if err != nil { - logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.String(), state.String()) + logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n", dep.Name(), state.String()) report.Skip(&Result{ - Dependency: dep.String(), + Dependency: dep.Name(), LicenseSpdxID: Unknown, Version: dep.Version, }) @@ -177,7 +173,7 @@ func (resolver *MavenPomResolver) ResolveDependencies(deps []*Dependency, config func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state *State, dep *Dependency, report *Report) error { result1, err1 := resolver.ResolveJar(config, state, filepath.Join(resolver.repo, dep.Path(), dep.Jar()), dep.Version) if result1 != nil { - result1.Dependency = dep.String() + result1.Dependency = dep.Name() report.Resolve(result1) return nil } @@ -188,7 +184,7 @@ func (resolver *MavenPomResolver) ResolveLicense(config *ConfigDeps, state *Stat return nil } - return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.String(), err1, err2) + return fmt.Errorf("failed to resolve license for <%s> from jar or pom: %+v, %+v", dep.Name(), err1, err2) } // ResolveLicenseFromPom search for license in the pom file, which may appear in the header comments or in license element of xml @@ -202,7 +198,7 @@ func (resolver *MavenPomResolver) ResolveLicenseFromPom(config *ConfigDeps, stat if pom != nil && len(pom.Licenses) != 0 { return &Result{ - Dependency: dep.String(), + Dependency: dep.Name(), LicenseFilePath: pomFile, LicenseContent: pom.Raw(), LicenseSpdxID: pom.AllLicenses(config), @@ -215,7 +211,7 @@ func (resolver *MavenPomResolver) ResolveLicenseFromPom(config *ConfigDeps, stat return nil, err } else if headerComments != "" { *state |= FoundLicenseInPomHeader - return resolver.IdentifyLicense(config, pomFile, dep.String(), headerComments, dep.Version) + return resolver.IdentifyLicense(config, pomFile, dep.Name(), headerComments, dep.Version) } return nil, fmt.Errorf("not found in pom file") @@ -287,7 +283,7 @@ func SeemLicense(content string) bool { return reMaybeLicense.MatchString(content) } -func LoadDependencies(data []byte) []*Dependency { +func LoadDependencies(data []byte, config *ConfigDeps) []*Dependency { depsTree := LoadDependenciesTree(data) cnt := 0 @@ -299,6 +295,9 @@ func LoadDependencies(data []byte) []*Dependency { queue := []*Dependency{} for _, depTree := range depsTree { + if config.IsExcluded(depTree.Name(), depTree.Version) { + continue + } queue = append(queue, depTree) for len(queue) > 0 { dep := queue[0] @@ -326,9 +325,8 @@ func LoadDependenciesTree(data []byte) []*Dependency { deps := make([]*Dependency, 0, len(rawDeps)) for _, rawDep := range rawDeps { - gid := strings.Split(string(rawDep[reFind.SubexpIndex("gid")]), ".") dep := &Dependency{ - GroupID: gid, + GroupID: string(rawDep[reFind.SubexpIndex("gid")]), ArtifactID: string(rawDep[reFind.SubexpIndex("aid")]), Packaging: string(rawDep[reFind.SubexpIndex("packaging")]), Version: string(rawDep[reFind.SubexpIndex("version")]), @@ -406,9 +404,8 @@ func (s *State) String() string { } type Dependency struct { - GroupID []string - ArtifactID, Version, Packaging, Scope string - TransitiveDeps []*Dependency + GroupID, ArtifactID, Version, Packaging, Scope string + TransitiveDeps []*Dependency } func (dep *Dependency) Clone() *Dependency { @@ -430,7 +427,7 @@ func (dep *Dependency) Count() int { } func (dep *Dependency) Path() string { - return fmt.Sprintf("%v/%v/%v", strings.Join(dep.GroupID, "/"), dep.ArtifactID, dep.Version) + return fmt.Sprintf("%v/%v/%v", strings.ReplaceAll(dep.GroupID, ".", "/"), dep.ArtifactID, dep.Version) } func (dep *Dependency) Pom() string { @@ -441,24 +438,8 @@ func (dep *Dependency) Jar() string { return fmt.Sprintf("%v-%v.jar", dep.ArtifactID, dep.Version) } -func (dep *Dependency) String() string { - buf := bytes.NewBuffer(nil) - w := bufio.NewWriter(buf) - - _, err := w.WriteString(fmt.Sprintf("%v:%v", strings.Join(dep.GroupID, "."), dep.ArtifactID)) - if err != nil { - logger.Log.Error(err) - } - - for _, tDep := range dep.TransitiveDeps { - _, err = w.WriteString(fmt.Sprintf("\n\t%v", tDep)) - if err != nil { - logger.Log.Error(err) - } - } - - _ = w.Flush() - return buf.String() +func (dep *Dependency) Name() string { + return fmt.Sprintf("%v:%v", dep.GroupID, dep.ArtifactID) } // PomFile is used to extract license from the pom.xml file diff --git a/pkg/deps/npm.go b/pkg/deps/npm.go index 4cdb28f..ee47c27 100644 --- a/pkg/deps/npm.go +++ b/pkg/deps/npm.go @@ -190,7 +190,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName, pkgPath string, conf Dependency: pkgName, } // resolve from the package.json file - if err := resolver.ResolvePkgFile(result, pkgPath, config.Licenses); err != nil { + if err := resolver.ResolvePkgFile(result, pkgPath, config); err != nil { result.ResolveErrors = append(result.ResolveErrors, err) } @@ -203,7 +203,7 @@ func (resolver *NpmResolver) ResolvePackageLicense(pkgName, pkgPath string, conf } // ResolvePkgFile tries to find and parse the package.json file to capture the license field -func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, licenses []*ConfigDepLicense) error { +func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, config *ConfigDeps) error { expectedPkgFile := filepath.Join(pkgPath, PkgFileName) packageInfo, err := resolver.ParsePkgFile(expectedPkgFile) if err != nil { @@ -211,13 +211,9 @@ func (resolver *NpmResolver) ResolvePkgFile(result *Result, pkgPath string, lice } result.Version = packageInfo.Version - for _, l := range licenses { - for _, version := range strings.Split(l.Version, ",") { - if l.Name == packageInfo.Name && version == packageInfo.Version { - result.LicenseSpdxID = l.License - return nil - } - } + if license, ok := config.GetUserConfiguredLicense(packageInfo.Name, packageInfo.Version); ok { + result.LicenseSpdxID = license + return nil } if lcs, ok := resolver.ResolveLicenseField(packageInfo.License); ok { @@ -287,13 +283,9 @@ func (resolver *NpmResolver) ResolveLcsFile(result *Result, pkgPath string, conf if result.LicenseSpdxID != "" { return nil } - for _, l := range config.Licenses { - for _, version := range strings.Split(l.Version, ",") { - if l.Name == info.Name() && version == result.Version { - result.LicenseSpdxID = l.License - return nil - } - } + if license, ok := config.GetUserConfiguredLicense(info.Name(), result.Version); ok { + result.LicenseSpdxID = license + return nil } identifier, err := license.Identify(string(content), config.Threshold) if err != nil { diff --git a/pkg/deps/result.go b/pkg/deps/result.go index 95a318b..66f87df 100644 --- a/pkg/deps/result.go +++ b/pkg/deps/result.go @@ -20,6 +20,7 @@ package deps import ( "fmt" "math" + "sort" "strings" ) @@ -56,6 +57,13 @@ func (report *Report) Skip(result *Result) { } func (report *Report) String() string { + sort.SliceStable(report.Resolved, func(i, j int) bool { + return report.Resolved[i].Dependency < report.Resolved[j].Dependency + }) + sort.SliceStable(report.Skipped, func(i, j int) bool { + return report.Skipped[i].Dependency < report.Skipped[j].Dependency + }) + dWidth, lWidth, vWidth := .0, .0, .0 for _, r := range report.Skipped { dWidth = math.Max(float64(len(r.Dependency)), dWidth)
