This is an automated email from the ASF dual-hosted git repository. kezhenxu94 pushed a commit to branch urls in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
commit cc09e30d7f2db67be3bc43fa3c3cd450a7d0c20b Author: kezhenxu94 <[email protected]> AuthorDate: Sun May 22 16:29:38 2022 +0800 Add custom license urls for identification --- assets/urls.yaml | 18 ++++++++++++++++++ go.mod | 2 ++ go.sum | 4 ++-- pkg/deps/jar.go | 19 ++++++++++++++----- pkg/deps/maven.go | 30 +++++++++++++++++++++--------- pkg/license/identifier.go | 19 ++++++++++++++++++- pkg/license/identifier_test.go | 30 ++++++++++++++++++++++++++++++ 7 files changed, 105 insertions(+), 17 deletions(-) diff --git a/assets/urls.yaml b/assets/urls.yaml new file mode 100644 index 0000000..8af29b0 --- /dev/null +++ b/assets/urls.yaml @@ -0,0 +1,18 @@ +Apache-2.0: + - apache.org/licenses/license-2.0 + - apache.org/licenses/license-2.0.txt + - apache.org/license/license-2.0 + - apache.org/license/license-2.0.txt + +MIT: + - bouncycastle.org/licence + - opensource.org/licenses/mit-license.php + +LGPL-2.1: + - gnu.org/licenses/old-licenses/lgpl-2.1 + +GPL-2.0-with-classpath-exception: + - gnu.org/software/classpath/license + +EPL-1.0: + - eclipse.org/legal/epl-v10 diff --git a/go.mod b/go.mod index 15ff201..fb294bb 100644 --- a/go.mod +++ b/go.mod @@ -21,3 +21,5 @@ require ( google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 ) + +replace github.com/google/licensecheck v0.3.1 => github.com/kezhenxu94/licensecheck v0.3.2-0.20220522082511-5121cbc45274 diff --git a/go.sum b/go.sum index ae2f5ee..09dd6b4 100644 --- a/go.sum +++ b/go.sum @@ -96,8 +96,6 @@ github.com/google/go-github/v33 v33.0.0/go.mod h1:GMdDnVZY/2TsWgp/lkYnpSAh6TrzhA github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/licensecheck v0.3.1 h1:QoxgoDkaeC4nFrtGN1jV7IPmDCHFNIVh54e5hSt6sPs= -github.com/google/licensecheck v0.3.1/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -117,6 +115,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kezhenxu94/licensecheck v0.3.2-0.20220522082511-5121cbc45274 h1:70F5y4oVIT9OoQ61ujeWgycYonXqW6kD6Z0LLi3V73c= +github.com/kezhenxu94/licensecheck v0.3.2-0.20220522082511-5121cbc45274/go.mod h1:ORkR35t/JjW+emNKtfJDII0zlciG9JgbT7SmsohlHmY= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/pkg/deps/jar.go b/pkg/deps/jar.go index 3c56188..8ab7296 100644 --- a/pkg/deps/jar.go +++ b/pkg/deps/jar.go @@ -90,11 +90,15 @@ func (resolver *JarResolver) ResolveJar(config *ConfigDeps, state *State, jarFil r := reSearchLicenseInManifestFile.FindStringSubmatch(content) if len(r) != 0 { + lcs := strings.TrimSpace(r[1]) + if l, err := license.Identify(lcs, config.Threshold); err == nil { + lcs = l + } report.Resolve(&Result{ Dependency: dep, LicenseFilePath: jarFile, LicenseContent: strings.TrimSpace(r[1]), - LicenseSpdxID: strings.TrimSpace(r[1]), + LicenseSpdxID: lcs, Version: version, }) return nil @@ -123,16 +127,21 @@ func (resolver *JarResolver) ReadFileFromZip(archiveFile *zip.File) (*bytes.Buff } func (resolver *JarResolver) IdentifyLicense(config *ConfigDeps, path, dep, content, version string, report *Report) error { - identifier, err := license.Identify(content, config.Threshold) - if err != nil { - return err + contents := strings.Split(content, "[, \\s]+") + identifiers := make([]string, 0, len(contents)) + for _, c := range contents { + identifier, err := license.Identify(c, config.Threshold) + if err != nil { + return err + } + identifiers = append(identifiers, identifier) } report.Resolve(&Result{ Dependency: dep, LicenseFilePath: path, LicenseContent: content, - LicenseSpdxID: identifier, + LicenseSpdxID: strings.Join(identifiers, " and "), Version: version, }) return nil diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go index 23c6324..bc665ec 100644 --- a/pkg/deps/maven.go +++ b/pkg/deps/maven.go @@ -32,6 +32,7 @@ import ( "golang.org/x/net/html/charset" "github.com/apache/skywalking-eyes/internal/logger" + "github.com/apache/skywalking-eyes/pkg/license" ) type MavenPomResolver struct { @@ -187,12 +188,14 @@ func (resolver *MavenPomResolver) ResolveLicenseFromPom(config *ConfigDeps, stat pom, err := resolver.ReadLicensesFromPom(pomFile) if err != nil { return err - } else if pom != nil && len(pom.Licenses) != 0 { + } + + if pom != nil && len(pom.Licenses) != 0 { report.Resolve(&Result{ Dependency: dep.Jar(), LicenseFilePath: pomFile, LicenseContent: pom.Raw(), - LicenseSpdxID: pom.AllLicenses(), + LicenseSpdxID: pom.AllLicenses(config), Version: dep.Version, }) @@ -391,7 +394,7 @@ func (s *State) String() string { m = append(m, "failed to resolve license content from manifest file found in jar") } - return strings.Join(m, "｜") + return strings.Join(m, " | ") } type Dependency struct { @@ -457,12 +460,12 @@ type PomFile struct { } // AllLicenses return all licenses found in pom.xml file -func (pom *PomFile) AllLicenses() string { +func (pom *PomFile) AllLicenses(config *ConfigDeps) string { licenses := []string{} for _, l := range pom.Licenses { - licenses = append(licenses, l.Item()) + licenses = append(licenses, l.Item(config)) } - return strings.Join(licenses, ", ") + return strings.Join(licenses, " and ") } // Raw return raw data @@ -481,14 +484,23 @@ type XMLLicense struct { Comments string `xml:"comments,omitempty"` } -func (l *XMLLicense) Item() string { - return GetLicenseFromURL(l.URL) +func (l *XMLLicense) Item(config *ConfigDeps) string { + if l.URL != "" { + return GetLicenseFromURL(l.URL, config) + } + if l.Name != "" { + return l.Name + } + return l.URL } func (l *XMLLicense) Raw() string { return fmt.Sprintf(`License: {Name: %s, URL: %s, Distribution: %s, Comments: %s, }`, l.Name, l.URL, l.Distribution, l.Comments) } -func GetLicenseFromURL(url string) string { +func GetLicenseFromURL(url string, config *ConfigDeps) string { + if l, err := license.Identify(url, config.Threshold); err == nil { + return l + } return url } diff --git a/pkg/license/identifier.go b/pkg/license/identifier.go index 5c42cf3..3609e03 100644 --- a/pkg/license/identifier.go +++ b/pkg/license/identifier.go @@ -24,6 +24,7 @@ import ( "sync" "github.com/google/licensecheck" + "gopkg.in/yaml.v3" "github.com/apache/skywalking-eyes/assets" "github.com/apache/skywalking-eyes/internal/logger" @@ -40,8 +41,24 @@ var ( // It will be initialized once. func scanner() *licensecheck.Scanner { scannerOnce.Do(func() { + licenses := licensecheck.BuiltinLicenses() + if bs, err := assets.Asset("urls.yaml"); err == nil { + licenseURLs := make(map[string][]string) + if err := yaml.Unmarshal(bs, &licenseURLs); err == nil { + logger.Log.Debug("license URLs:", licenseURLs) + for id, urls := range licenseURLs { + for _, url := range urls { + licenses = append(licenses, licensecheck.License{ + ID: id, + URL: url, + Type: licensecheck.Unknown, + }) + } + } + } + } var err error - _scanner, err = licensecheck.NewScanner(licensecheck.BuiltinLicenses()) + _scanner, err = licensecheck.NewScanner(licenses) if err != nil { logger.Log.Fatalf("licensecheck.NewScanner: %v", err) } diff --git a/pkg/license/identifier_test.go b/pkg/license/identifier_test.go index ae5b666..7ac706f 100644 --- a/pkg/license/identifier_test.go +++ b/pkg/license/identifier_test.go @@ -323,6 +323,36 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. `, want: "BSD-3-Clause", }, + { + name: "URL http://www.apache.org/licenses/LICENSE-2.0";, + content: "http://www.apache.org/licenses/LICENSE-2.0";, + want: "Apache-2.0", + }, + { + name: "URL https://opensource.org/licenses/Apache-2.0";, + content: "https://opensource.org/licenses/Apache-2.0";, + want: "Apache-2.0", + }, + { + name: "URL http://www.apache.org/licenses/LICENSE-2.0.txt";, + content: "http://www.apache.org/licenses/LICENSE-2.0.txt";, + want: "Apache-2.0", + }, + { + name: "URL https://www.bouncycastle.org/licence.html";, + content: "https://www.bouncycastle.org/licence.html";, + want: "MIT", + }, + { + name: "URL https://www.bouncycastle.org/licence.html";, + content: "http://www.gnu.org/software/classpath/license.html, http://opensource.org/licenses/MIT";, + want: "GPL-2.0-with-classpath-exception and MIT", + }, + { + name: "URL", + content: "http://www.mozilla.org/MPL/MPL-1.1.html, http://www.gnu.org/licenses/lgpl-2.1.html";, + want: "MPL-1.1 and LGPL-2.1", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {
