This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-eyes.git
The following commit(s) were added to refs/heads/main by this push:
new 03dbef4 Add support for resolving jars' licenses (#53)
03dbef4 is described below
commit 03dbef403159bb04fa3247b5f125458d830d69ef
Author: MoGuGuai-hzr <[email protected]>
AuthorDate: Thu Aug 12 13:22:55 2021 +0800
Add support for resolving jars' licenses (#53)
---
pkg/deps/jar.go | 136 ++++++++++++++++++++++++++++++++
pkg/deps/{maven_test.go => jar_test.go} | 103 +++++++++++++++---------
pkg/deps/maven.go | 131 +-----------------------------
pkg/deps/maven_test.go | 37 +++++++--
4 files changed, 234 insertions(+), 173 deletions(-)
diff --git a/pkg/deps/jar.go b/pkg/deps/jar.go
new file mode 100644
index 0000000..d7b851d
--- /dev/null
+++ b/pkg/deps/jar.go
@@ -0,0 +1,136 @@
+//
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package deps
+
+import (
+ "archive/zip"
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "path/filepath"
+ "regexp"
+
+ "github.com/apache/skywalking-eyes/license-eye/internal/logger"
+ "github.com/apache/skywalking-eyes/license-eye/pkg/license"
+)
+
+type JarResolver struct{}
+
+func (resolver *JarResolver) CanResolve(jarFile string) bool {
+ return filepath.Ext(jarFile) == ".jar"
+}
+
+func (resolver *JarResolver) Resolve(jarFile string, report *Report) error {
+ state := NotFound
+ if err := resolver.ResolveJar(&state, jarFile, report); err != nil {
+ dep := filepath.Base(jarFile)
+ logger.Log.Warnf("Failed to resolve the license of <%s>: %v\n",
dep, state.String())
+ report.Skip(&Result{
+ Dependency: dep,
+ LicenseSpdxID: Unknown,
+ })
+ }
+
+ return nil
+}
+
+func (resolver *JarResolver) ResolveJar(state *State, jarFile string, report
*Report) error {
+ dep := filepath.Base(jarFile)
+
+ compressedJar, err := zip.OpenReader(jarFile)
+ if err != nil {
+ return err
+ }
+ defer compressedJar.Close()
+
+ var manifestFile *zip.File
+
+ // traverse all files in jar
+ for _, compressedFile := range compressedJar.File {
+ archiveFile := compressedFile.Name
+ switch {
+ case reHaveManifestFile.MatchString(archiveFile):
+ manifestFile = compressedFile
+
+ case possibleLicenseFileName.MatchString(archiveFile):
+ *state |= FoundLicenseInJarLicenseFile
+ buf, err := resolver.ReadFileFromZip(compressedFile)
+ if err != nil {
+ return err
+ }
+
+ return resolver.IdentifyLicense(jarFile, dep,
buf.String(), report)
+ }
+ }
+
+ if manifestFile != nil {
+ buf, err := resolver.ReadFileFromZip(manifestFile)
+ if err != nil {
+ return err
+ }
+ norm := regexp.MustCompile(`(?im)[\r\n]+ +`)
+ content := norm.ReplaceAllString(buf.String(), "")
+
+ r := reSearchLicenseInManifestFile.FindStringSubmatch(content)
+ if len(r) != 0 {
+ report.Resolve(&Result{
+ Dependency: dep,
+ LicenseFilePath: jarFile,
+ LicenseContent: r[1],
+ LicenseSpdxID: r[1],
+ })
+ return nil
+ }
+ }
+
+ return fmt.Errorf("cannot find license content")
+}
+
+func (resolver *JarResolver) ReadFileFromZip(archiveFile *zip.File)
(*bytes.Buffer, error) {
+ file, err := archiveFile.Open()
+ if err != nil {
+ return nil, err
+ }
+
+ buf := bytes.NewBuffer(nil)
+ w := bufio.NewWriter(buf)
+ _, err = io.CopyN(w, file, int64(archiveFile.UncompressedSize64))
+ if err != nil {
+ return nil, err
+ }
+
+ w.Flush()
+ file.Close()
+ return buf, nil
+}
+
+func (resolver *JarResolver) IdentifyLicense(path, dep, content string, report
*Report) error {
+ identifier, err := license.Identify(path, content)
+ if err != nil {
+ return err
+ }
+
+ report.Resolve(&Result{
+ Dependency: dep,
+ LicenseFilePath: path,
+ LicenseContent: content,
+ LicenseSpdxID: identifier,
+ })
+ return nil
+}
diff --git a/pkg/deps/maven_test.go b/pkg/deps/jar_test.go
similarity index 51%
copy from pkg/deps/maven_test.go
copy to pkg/deps/jar_test.go
index d63d519..4d55f89 100644
--- a/pkg/deps/maven_test.go
+++ b/pkg/deps/jar_test.go
@@ -18,59 +18,80 @@
package deps_test
import (
- "bufio"
+ "fmt"
+ "io/ioutil"
"os"
+ "os/exec"
"path/filepath"
"testing"
+ "github.com/apache/skywalking-eyes/license-eye/internal/logger"
"github.com/apache/skywalking-eyes/license-eye/pkg/deps"
)
-func TestCanResolve(t *testing.T) {
- resolver := new(deps.MavenPomResolver)
+func TestCanResolveJarFile(t *testing.T) {
+ resolver := new(deps.JarResolver)
for _, test := range []struct {
fileName string
exp bool
}{
- {"pom.xml", true},
- {"POM.XML", false},
- {"log4j-1.2.12.pom", false},
- {".pom", false},
+ {"1.jar", true},
+ {"/tmp/1.jar", true},
+ {"1.jar2", false},
+ {"protobuf-java-3.13.0.jar", true},
+ {"slf4j-api-1.7.25.jar", true},
} {
b := resolver.CanResolve(test.fileName)
if b != test.exp {
- t.Errorf("MavenPomResolver.CanResolve(\"%v\") = %v,
want %v", test.fileName, b, test.exp)
+ t.Errorf("JarResolver.CanResolve(\"%v\") = %v, want
%v", test.fileName, b, test.exp)
}
}
}
-func dump(fileName, content string) error {
- file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
0666)
- if err != nil {
- return err
+func copyJars(t *testing.T, pomFile, content string) ([]string, error) {
+ dir := filepath.Dir(pomFile)
+
+ if err := os.Chdir(dir); err != nil {
+ return nil, err
+ }
+
+ if err := dumpPomFile(pomFile, content); err != nil {
+ return nil, err
}
- defer file.Close()
- write := bufio.NewWriter(file)
- _, err = write.WriteString(content)
+ if _, err := exec.Command("mvn", "dependency:copy-dependencies",
"-DoutputDirectory=./lib", "-DincludeScope=runtime").Output(); err != nil {
+ return nil, err
+ }
+
+ jars := []string{}
+ files, err := ioutil.ReadDir(filepath.Join(dir, "lib"))
if err != nil {
- return err
+ return nil, err
+ }
+
+ for _, file := range files {
+ if !file.IsDir() {
+ jars = append(jars, filepath.Join(dir, "lib",
file.Name()))
+ }
}
- write.Flush()
- return nil
+ return jars, nil
}
-func TestResolve(t *testing.T) {
- resolver := new(deps.MavenPomResolver)
+func TestResolveJar(t *testing.T) {
+ if _, err := exec.Command("mvn", "--version").Output(); err != nil {
+ logger.Log.Warnf("Failed to find mvn, the test `TestResolveJar`
was skipped")
+ return
+ }
+
+ resolver := new(deps.JarResolver)
- tempDir := deps.NewTempDirGenerator()
- path, err := tempDir.Create()
+ path, err := tmpDir()
if err != nil {
t.Error(err)
return
}
- defer tempDir.Destroy()
+ defer destroyTmpDir(t, path)
pomFile := filepath.Join(path, "pom.xml")
@@ -100,28 +121,34 @@ func TestResolve(t *testing.T) {
<artifactId>commons-logging</artifactId>
<version>1.2</version>
</dependency>
- <!--
https://mvnrepository.com/artifact/org.apache.skywalking/skywalking-sharing-server-plugin
-->
+ <!--
https://mvnrepository.com/artifact/org.apache.commons/commons-math3 -->
<dependency>
- <groupId>org.apache.skywalking</groupId>
-
<artifactId>skywalking-sharing-server-plugin</artifactId>
- <version>8.6.0</version>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
</dependency>
</dependencies>
- </project>`, 107},
+ </project>`, 4},
} {
- dump(pomFile, test.pomContent)
+ jars, err := copyJars(t, pomFile, test.pomContent)
+ if err != nil {
+ t.Error(err)
+ return
+ }
- if resolver.CanResolve(pomFile) {
- report := deps.Report{}
- if err := resolver.Resolve(pomFile, &report); err !=
nil {
- t.Error(err)
- return
- }
+ report := deps.Report{}
+ for _, jar := range jars {
+ if resolver.CanResolve(jar) {
+ if err := resolver.Resolve(jar, &report); err
!= nil {
+ t.Error(err)
+ return
+ }
- if len(report.Resolved)+len(report.Skipped) != test.cnt
{
- t.Errorf("the expected number of jar packages
is: %d, but actually: %d. result:\n%v", test.cnt,
len(report.Resolved)+len(report.Skipped), report.String())
}
-
}
+ if len(report.Resolved)+len(report.Skipped) != test.cnt {
+ t.Errorf("the expected number of jar packages is: %d,
but actually: %d. result:\n%v", test.cnt,
len(report.Resolved)+len(report.Skipped), report.String())
+ }
+ fmt.Println(report.String())
}
}
diff --git a/pkg/deps/maven.go b/pkg/deps/maven.go
index 4a6c476..7f9d05c 100644
--- a/pkg/deps/maven.go
+++ b/pkg/deps/maven.go
@@ -18,13 +18,11 @@
package deps
import (
- "archive/zip"
"bufio"
"bytes"
"encoding/xml"
"fmt"
"io"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -34,10 +32,10 @@ import (
"golang.org/x/net/html/charset"
"github.com/apache/skywalking-eyes/license-eye/internal/logger"
- "github.com/apache/skywalking-eyes/license-eye/pkg/license"
)
type MavenPomResolver struct {
+ JarResolver
maven string
repo string
}
@@ -165,7 +163,7 @@ func (resolver *MavenPomResolver) ResolveDependencies(deps
[]*Dependency, report
// ResolveLicense search all possible locations of the license, such as pom
file, jar package
func (resolver *MavenPomResolver) ResolveLicense(state *State, dep
*Dependency, report *Report) error {
- err := resolver.ResolveLicenseFromJar(state, dep, report)
+ err := resolver.ResolveJar(state, filepath.Join(resolver.repo,
dep.Path(), dep.Jar()), report)
if err == nil {
return nil
}
@@ -183,7 +181,7 @@ func (resolver *MavenPomResolver)
ResolveLicenseFromPom(state *State, dep *Depen
} else if pom != nil && len(pom.Licenses) != 0 {
report.Resolve(&Result{
Dependency: dep.Jar(),
- LicenseFilePath: dep.Path(),
+ LicenseFilePath: pomFile,
LicenseContent: pom.Raw(),
LicenseSpdxID: pom.AllLicenses(),
})
@@ -196,7 +194,7 @@ func (resolver *MavenPomResolver)
ResolveLicenseFromPom(state *State, dep *Depen
return err
} else if headerComments != "" {
*state |= FoundLicenseInPomHeader
- return resolver.IdentifyLicense(dep, headerComments, report)
+ return resolver.IdentifyLicense(pomFile, dep.Jar(),
headerComments, report)
}
return fmt.Errorf("not found in pom file")
@@ -268,91 +266,6 @@ func SeemLicense(content string) bool {
return reMaybeLicense.MatchString(content)
}
-// ResolveLicenseFromJar search for the license in the jar package, and it may
appear in MANIFEST.MF, LICENSE.txt, NOTICE.txt
-func (resolver *MavenPomResolver) ResolveLicenseFromJar(state *State, dep
*Dependency, report *Report) (err error) {
- jarPath := filepath.Join(resolver.repo, dep.Path(), dep.Jar())
- compressedJar, err := zip.OpenReader(jarPath)
- if err != nil {
- return err
- }
- defer compressedJar.Close()
-
- var manifestFile *zip.File
-
- // traverse all files in jar
- for _, compressedFile := range compressedJar.File {
- archiveFile := compressedFile.Name
- switch {
- case reHaveManifestFile.MatchString(archiveFile):
- manifestFile = compressedFile
-
- case possibleLicenseFileName.MatchString(archiveFile):
- *state |= FoundLicenseInJarLicenseFile
- buf, err := resolver.ReadFileFromZip(compressedFile)
- if err != nil {
- return err
- }
-
- return resolver.IdentifyLicense(dep, buf.String(),
report)
- }
- }
-
- if manifestFile != nil {
- buf, err := resolver.ReadFileFromZip(manifestFile)
- if err != nil {
- return err
- }
- norm := regexp.MustCompile(`(?im)[\r\n]+ +`)
- content := norm.ReplaceAllString(buf.String(), "")
-
- r := reSearchLicenseInManifestFile.FindStringSubmatch(content)
- if len(r) != 0 {
- report.Resolve(&Result{
- Dependency: dep.Jar(),
- LicenseFilePath: dep.Path(),
- LicenseContent: r[1],
- LicenseSpdxID: r[1],
- })
- return nil
- }
- }
-
- return fmt.Errorf("cannot find license content")
-}
-
-func (resolver *MavenPomResolver) ReadFileFromZip(archiveFile *zip.File)
(*bytes.Buffer, error) {
- file, err := archiveFile.Open()
- if err != nil {
- return nil, err
- }
-
- buf := bytes.NewBuffer(nil)
- w := bufio.NewWriter(buf)
- _, err = io.CopyN(w, file, int64(archiveFile.UncompressedSize64))
- if err != nil {
- return nil, err
- }
-
- w.Flush()
- file.Close()
- return buf, nil
-}
-
-func (resolver *MavenPomResolver) IdentifyLicense(dep *Dependency, content
string, report *Report) error {
- identifier, err := license.Identify(dep.Path(), content)
- if err != nil {
- return err
- }
-
- report.Resolve(&Result{
- Dependency: dep.Jar(),
- LicenseFilePath: dep.Path(),
- LicenseContent: content,
- LicenseSpdxID: identifier,
- })
- return nil
-}
-
func LoadDependencies(data []byte) []*Dependency {
depsTree := LoadDependenciesTree(data)
@@ -527,42 +440,6 @@ func (dep *Dependency) String() string {
return buf.String()
}
-// TempDirGenerator Create and destroy one temporary directory
-type TempDirGenerator interface {
- Create() (string, error)
- Destroy()
-}
-
-func NewTempDirGenerator() TempDirGenerator {
- return new(TempDir)
-}
-
-// TempDir an implementation of the TempDirGenerator
-type TempDir struct {
- dir string
-}
-
-func (t *TempDir) Create() (string, error) {
- tmpDir, err := ioutil.TempDir("", "")
- if err != nil {
- return "", err
- }
- t.dir = tmpDir
- return tmpDir, nil
-}
-
-func (t *TempDir) Destroy() {
- if t.dir == "" {
- logger.Log.Errorf("the temporary directory does not exist")
- return
- }
-
- err := os.RemoveAll(t.dir)
- if err != nil {
- logger.Log.Errorln(err)
- }
-}
-
// PomFile is used to extract license from the pom.xml file
type PomFile struct {
XMLName xml.Name `xml:"project"`
diff --git a/pkg/deps/maven_test.go b/pkg/deps/maven_test.go
index d63d519..906f7a3 100644
--- a/pkg/deps/maven_test.go
+++ b/pkg/deps/maven_test.go
@@ -19,6 +19,8 @@ package deps_test
import (
"bufio"
+ "fmt"
+ "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -26,7 +28,7 @@ import (
"github.com/apache/skywalking-eyes/license-eye/pkg/deps"
)
-func TestCanResolve(t *testing.T) {
+func TestCanResolvePomFile(t *testing.T) {
resolver := new(deps.MavenPomResolver)
for _, test := range []struct {
fileName string
@@ -44,7 +46,7 @@ func TestCanResolve(t *testing.T) {
}
}
-func dump(fileName, content string) error {
+func dumpPomFile(fileName, content string) error {
file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_TRUNC|os.O_CREATE,
0666)
if err != nil {
return err
@@ -61,16 +63,35 @@ func dump(fileName, content string) error {
return nil
}
-func TestResolve(t *testing.T) {
+func tmpDir() (string, error) {
+ dir, err := ioutil.TempDir("", "")
+ if err != nil {
+ return "", err
+ }
+ return dir, nil
+}
+
+func destroyTmpDir(t *testing.T, dir string) {
+ if dir == "" {
+ t.Errorf("the temporary directory does not exist")
+ return
+ }
+
+ err := os.RemoveAll(dir)
+ if err != nil {
+ t.Error(err)
+ }
+}
+
+func TestResolveMaven(t *testing.T) {
resolver := new(deps.MavenPomResolver)
- tempDir := deps.NewTempDirGenerator()
- path, err := tempDir.Create()
+ path, err := tmpDir()
if err != nil {
t.Error(err)
return
}
- defer tempDir.Destroy()
+ defer destroyTmpDir(t, path)
pomFile := filepath.Join(path, "pom.xml")
@@ -109,7 +130,7 @@ func TestResolve(t *testing.T) {
</dependencies>
</project>`, 107},
} {
- dump(pomFile, test.pomContent)
+ dumpPomFile(pomFile, test.pomContent)
if resolver.CanResolve(pomFile) {
report := deps.Report{}
@@ -121,7 +142,7 @@ func TestResolve(t *testing.T) {
if len(report.Resolved)+len(report.Skipped) != test.cnt
{
t.Errorf("the expected number of jar packages
is: %d, but actually: %d. result:\n%v", test.cnt,
len(report.Resolved)+len(report.Skipped), report.String())
}
-
+ fmt.Println(report.String())
}
}
}
