This is an automated email from the ASF dual-hosted git repository.
wusheng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/skywalking-rover.git
The following commit(s) were added to refs/heads/main by this push:
new 516c080 Introduce the BTFhub module (#35)
516c080 is described below
commit 516c0802badd003014371a198b8159b9fb5c725b
Author: mrproliu <[email protected]>
AuthorDate: Sat May 21 16:27:53 2022 +0800
Introduce the BTFhub module (#35)
---
.github/workflows/rover.yaml | 2 +
.gitignore | 4 +-
CHANGES.md | 1 +
Makefile | 1 +
dist/LICENSE | 3 +-
dist/LICENSE.tpl | 3 +
docker/Dockerfile.build | 2 +-
pkg/profiling/task/offcpu/runner.go | 5 +-
pkg/profiling/task/oncpu/runner.go | 2 +-
pkg/tools/btf/check.go | 58 +++++++++++++++
pkg/tools/btf/ebpf.go | 96 ++++++++++++++++++++++++
pkg/tools/operator/distribution.go | 136 ++++++++++++++++++++++++++++++++++
pkg/tools/operator/uname.go | 56 ++++++++++++++
scripts/build/bash/btfgen.sh | 50 +++++++++++++
scripts/build/btf.mk | 26 +++++++
scripts/release/create_bin_release.sh | 2 +-
16 files changed, 440 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/rover.yaml b/.github/workflows/rover.yaml
index 11c2784..8c32a29 100644
--- a/.github/workflows/rover.yaml
+++ b/.github/workflows/rover.yaml
@@ -39,6 +39,8 @@ jobs:
run: make deps
- name: Test
run: make container-test
+ - name: BTF Generate
+ run: make container-btfgen
- name: Lint
run: make container-lint
- name: Make binary
diff --git a/.gitignore b/.gitignore
index ea40d08..b24b3ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,6 @@ coverage.txt
!/dist/bin/
pkg/**/bpf_*.o
-pkg/**/bpf_*.go
\ No newline at end of file
+pkg/**/bpf_*.go
+
+pkg/tools/btf/files/**/*.btf
\ No newline at end of file
diff --git a/CHANGES.md b/CHANGES.md
index a6812cc..aaa79e3 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,6 +6,7 @@ Release Notes.
------------------
#### Features
* Support `OFF_CPU` Profiling.
+* Introduce the `BTFHub` module.
#### Bug Fixes
diff --git a/Makefile b/Makefile
index e4b798e..c20373b 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,7 @@
include scripts/build/base.mk
include scripts/build/generate.mk
+include scripts/build/btf.mk
include scripts/build/test.mk
include scripts/build/lint.mk
include scripts/build/build.mk
diff --git a/dist/LICENSE b/dist/LICENSE
index 48dc477..25c53cb 100644
--- a/dist/LICENSE
+++ b/dist/LICENSE
@@ -425,7 +425,8 @@ The text of each license is also included at
licenses/license-[project].txt.
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 Apache-2.0
k8s.io/utils v0.0.0-20211116205334-6203023598ed Apache-2.0
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 Apache-2.0
- skywalking.apache.org/repo/goapi v0.0.0-20220513074115-4af2c2d37d2f
Apache-2.0
+ skywalking.apache.org/repo/goapi v0.0.0-20220518063910-af3e2df60bce
Apache-2.0
+ github.com/aquasecurity/btfhub-archive v0.1 Apache-2.0
========================================================================
MIT licenses
diff --git a/dist/LICENSE.tpl b/dist/LICENSE.tpl
index 8429b77..c728de8 100644
--- a/dist/LICENSE.tpl
+++ b/dist/LICENSE.tpl
@@ -43,4 +43,7 @@ The text of each license is also included at
licenses/license-[project].txt.
{{range .Deps}}
{{.Name}} {{.Version}} {{.LicenseID}}
{{- end }}
+{{- if eq .LicenseID "Apache-2.0"}}
+ github.com/aquasecurity/btfhub-archive v0.1 Apache-2.0
+{{- end}}
{{ end }}
\ No newline at end of file
diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build
index a04305f..03e1be9 100644
--- a/docker/Dockerfile.build
+++ b/docker/Dockerfile.build
@@ -25,7 +25,7 @@ COPY . .
ENV CGO_ENABLED=0
-RUN VERSION=$VERSION make generate && make linux
+RUN VERSION=$VERSION make btfgen && make linux
RUN mv /src/bin/skywalking-rover-${VERSION}-linux-amd64
/src/bin/skywalking-rover
FROM alpine
diff --git a/pkg/profiling/task/offcpu/runner.go
b/pkg/profiling/task/offcpu/runner.go
index 85fea37..81bc003 100644
--- a/pkg/profiling/task/offcpu/runner.go
+++ b/pkg/profiling/task/offcpu/runner.go
@@ -32,6 +32,7 @@ import (
"github.com/apache/skywalking-rover/pkg/process/api"
"github.com/apache/skywalking-rover/pkg/profiling/task/base"
"github.com/apache/skywalking-rover/pkg/tools"
+ "github.com/apache/skywalking-rover/pkg/tools/btf"
"github.com/apache/skywalking-rover/pkg/tools/profiling"
v3 "skywalking.apache.org/repo/goapi/collect/ebpf/profiling/v3"
@@ -39,7 +40,7 @@ import (
// $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
// nolint
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags
$BPF_CFLAGS bpf $REPO_ROOT/bpf/profiling/offcpu.c -- -I$REPO_ROOT/bpf/include
-D__TARGET_ARCH_x86
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target bpfel -cc
$BPF_CLANG -cflags $BPF_CFLAGS bpf $REPO_ROOT/bpf/profiling/offcpu.c --
-I$REPO_ROOT/bpf/include -D__TARGET_ARCH_x86
var log = logger.GetLogger("profiling", "task", "offcpu")
@@ -105,7 +106,7 @@ func (r *Runner) Run(ctx context.Context, notify
base.ProfilingRunningSuccessNot
if !replacedPid {
return fmt.Errorf("replace the monitor pid failure")
}
- if err1 := spec.LoadAndAssign(&objs, nil); err1 != nil {
+ if err1 := spec.LoadAndAssign(&objs,
btf.GetEBPFCollectionOptionsIfNeed()); err1 != nil {
return err1
}
r.bpf = &objs
diff --git a/pkg/profiling/task/oncpu/runner.go
b/pkg/profiling/task/oncpu/runner.go
index 6ea9f47..7c9cc33 100644
--- a/pkg/profiling/task/oncpu/runner.go
+++ b/pkg/profiling/task/oncpu/runner.go
@@ -47,7 +47,7 @@ import (
// $BPF_CLANG and $BPF_CFLAGS are set by the Makefile.
// nolint
-//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc $BPF_CLANG -cflags
$BPF_CFLAGS bpf $REPO_ROOT/bpf/profiling/oncpu.c -- -I$REPO_ROOT/bpf/include
+//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -target bpfel -cc
$BPF_CLANG -cflags $BPF_CFLAGS bpf $REPO_ROOT/bpf/profiling/oncpu.c --
-I$REPO_ROOT/bpf/include
var log = logger.GetLogger("profiling", "task", "oncpu")
diff --git a/pkg/tools/btf/check.go b/pkg/tools/btf/check.go
new file mode 100644
index 0000000..9cc3afe
--- /dev/null
+++ b/pkg/tools/btf/check.go
@@ -0,0 +1,58 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package btf
+
+import (
+ "debug/elf"
+ "fmt"
+
+ "github.com/apache/skywalking-rover/pkg/tools/host"
+ "github.com/apache/skywalking-rover/pkg/tools/operator"
+ "github.com/apache/skywalking-rover/pkg/tools/path"
+)
+
+func ExistKernelBTF() (string, error) {
+ if path.Exists(host.GetFileInHost("/sys/kernel/btf/vmlinux")) {
+ return "/sys/kernel/btf/vmlinux", nil
+ }
+
+ uname, err := operator.GetOSUname()
+ if err != nil {
+ return "", err
+ }
+
+ // use same list of locations as libbpf
+ //
https://github.com/libbpf/libbpf/blob/9a3a42608dbe3731256a5682a125ac1e23bced8f/src/btf.c#L3114-L3122
+ locations := []string{
+ "/boot/vmlinux-%s",
+ "/lib/modules/%s/vmlinux-%[1]s",
+ "/lib/modules/%s/build/vmlinux",
+ "/usr/lib/modules/%s/kernel/vmlinux",
+ "/usr/lib/debug/boot/vmlinux-%s",
+ "/usr/lib/debug/boot/vmlinux-%s.debug",
+ "/usr/lib/debug/lib/modules/%s/vmlinux",
+ }
+
+ for _, loc := range locations {
+ _, err := elf.Open(host.GetFileInHost(fmt.Sprintf(loc,
uname.Release)))
+ if err == nil {
+ return loc, nil
+ }
+ }
+ return "", fmt.Errorf("could not found")
+}
diff --git a/pkg/tools/btf/ebpf.go b/pkg/tools/btf/ebpf.go
new file mode 100644
index 0000000..f44e7d6
--- /dev/null
+++ b/pkg/tools/btf/ebpf.go
@@ -0,0 +1,96 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package btf
+
+import (
+ "bytes"
+ "embed"
+ "fmt"
+ "path/filepath"
+ "sync"
+
+ "github.com/apache/skywalking-rover/pkg/logger"
+ "github.com/apache/skywalking-rover/pkg/tools/operator"
+
+ "github.com/cilium/ebpf"
+)
+
+//go:embed *
+var assets embed.FS
+
+var (
+ customizedBTFData []byte
+ findBTFOnce sync.Once
+
+ log = logger.GetLogger("tools", "btf")
+)
+
+func GetEBPFCollectionOptionsIfNeed() *ebpf.CollectionOptions {
+ findBTFOnce.Do(func() {
+ btfPath, isCustomizedBTF, err := getKernelBTFAddress()
+ if err != nil {
+ log.Warnf("found BTF failure: %v", err)
+ return
+ }
+
+ if !isCustomizedBTF {
+ return
+ }
+ d, err := asset(btfPath)
+ if err != nil {
+ log.Warnf("could not found the customized BTF file:
%s", btfPath)
+ return
+ }
+ customizedBTFData = d
+ })
+
+ if customizedBTFData == nil {
+ return nil
+ }
+
+ return &ebpf.CollectionOptions{Programs: ebpf.ProgramOptions{TargetBTF:
bytes.NewReader(customizedBTFData)}}
+}
+
+// getKernelBTFAddress means get the kernel BTF file path
+func getKernelBTFAddress() (btfPath string, isCustomizedBTF bool, err error) {
+ path, err := ExistKernelBTF()
+ if err == nil {
+ return path, false, nil
+ }
+
+ distributeInfo, err := operator.GetDistributionInfo()
+ if err != nil {
+ return "", false, fmt.Errorf("could not load the system
distribute info: %v", err)
+ }
+ uname, err := operator.GetOSUname()
+ if err != nil {
+ return "", false, fmt.Errorf("could not load the uname info:
%v", err)
+ }
+
+ btfPath = fmt.Sprintf("files/%s/%s/%s/%s.btf", distributeInfo.Name,
distributeInfo.Version,
+ distributeInfo.Architecture, uname.Release)
+ _, err = asset(btfPath)
+ if err != nil {
+ return "", true, fmt.Errorf("could not found customized BTF
file: %s", btfPath)
+ }
+ return btfPath, true, nil
+}
+
+func asset(file string) ([]byte, error) {
+ return assets.ReadFile(filepath.ToSlash(file))
+}
diff --git a/pkg/tools/operator/distribution.go
b/pkg/tools/operator/distribution.go
new file mode 100644
index 0000000..c63831e
--- /dev/null
+++ b/pkg/tools/operator/distribution.go
@@ -0,0 +1,136 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package operator
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "strings"
+
+ "github.com/apache/skywalking-rover/pkg/tools/host"
+)
+
+type DistributionInfo struct {
+ Name string // ubuntu, debian, etcd.
+ Version string // 20.04, 7, etcd.
+ Architecture string // x86_64, etcd.
+}
+
+// GetDistributionInfo of machine
+func GetDistributionInfo() (*DistributionInfo, error) {
+ var result = &DistributionInfo{}
+ tryingToFindDistributionByReleaseFile(result, "/etc/lsb-release",
"DISTRIB_ID", "DISTRIB_RELEASE", "")
+ tryingToFindDistributionByReleaseFile(result, "/etc/os-release", "ID",
"VERSION_ID", "")
+
+ tryingToFindDistributionByCommand(result, "Distributor ID", "Release",
"", "lsb_release", "-a")
+ tryingToFindDistributionByCommand(result, "", "", "Architecture",
"hostnamectl")
+ tryingToFindDistributionByCommand(result, "", "", "Architecture", "")
+
+ // fix arch
+ fixArchByCommand(result, "arch")
+ fixArchByCommand(result, "uname", "-m")
+
+ if !result.AllDataSuccess() {
+ return nil, fmt.Errorf("incomplete data, current existing data:
%v", result)
+ }
+ result.Normalize()
+ return result, nil
+}
+
+func tryingToFindDistributionByReleaseFile(data *DistributionInfo, filename,
nameKey, versionKey, architectureKey string) {
+ if data.AllDataSuccess() {
+ return
+ }
+ file, err := os.Open(host.GetFileInHost(filename))
+ if err != nil {
+ return
+ }
+ content := splitData(file, "=")
+ fixData(data, content, nameKey, versionKey, architectureKey)
+}
+
+func tryingToFindDistributionByCommand(data *DistributionInfo, nameKey,
versionKey, architectureKey, commandName string, args ...string) {
+ if data.AllDataSuccess() {
+ return
+ }
+ output := executeCommand(commandName, args...)
+ content := splitData(bytes.NewReader(output), ":")
+ fixData(data, content, nameKey, versionKey, architectureKey)
+}
+
+func executeCommand(name string, args ...string) []byte {
+ c := exec.Command(name, args...)
+ output, err := c.CombinedOutput()
+ if err == nil {
+ return output
+ }
+ return nil
+}
+
+func fixArchByCommand(data *DistributionInfo, commandName string, args
...string) {
+ if data.Architecture != "" {
+ return
+ }
+ data.Architecture = string(executeCommand(commandName, args...))
+}
+
+func fixData(d *DistributionInfo, content map[string]string, name, version,
architecture string) {
+ if d.Name == "" && content[name] != "" {
+ d.Name = content[name]
+ }
+ if d.Version == "" && content[version] != "" {
+ d.Version = content[version]
+ }
+ if d.Architecture == "" && content[architecture] != "" {
+ d.Architecture = content[architecture]
+ }
+}
+
+func splitData(r io.Reader, dataSplit string) map[string]string {
+ scanner := bufio.NewScanner(r)
+ result := make(map[string]string)
+ for scanner.Scan() {
+ split := strings.Split(scanner.Text(), dataSplit)
+ if len(split) != 2 {
+ continue
+ }
+ result[strings.TrimSpace(split[0])] =
strings.TrimSpace(split[1])
+ }
+ return result
+}
+
+func (d *DistributionInfo) AllDataSuccess() bool {
+ return d.Name != "" && d.Version != "" && d.Architecture != ""
+}
+
+func (d *DistributionInfo) Normalize() {
+ d.Name = normalizeContent(d.Name)
+ d.Version = normalizeContent(d.Version)
+ d.Architecture = normalizeContent(d.Architecture)
+}
+
+func normalizeContent(d string) string {
+ d = strings.TrimSpace(d)
+ d = strings.ToLower(d)
+ d = strings.ReplaceAll(d, "-", "_")
+ return d
+}
diff --git a/pkg/tools/operator/uname.go b/pkg/tools/operator/uname.go
new file mode 100644
index 0000000..c52806f
--- /dev/null
+++ b/pkg/tools/operator/uname.go
@@ -0,0 +1,56 @@
+// Licensed to Apache Software Foundation (ASF) under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Apache Software Foundation (ASF) licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package operator
+
+import "golang.org/x/sys/unix"
+
+type UnameInfo struct {
+ SysName string
+ Nodename string
+ Release string
+ Version string
+ Machine string
+}
+
+func GetOSUname() (*UnameInfo, error) {
+ u := unix.Utsname{}
+ e := unix.Uname(&u)
+ if e != nil {
+ return nil, e
+ }
+ ui := UnameInfo{}
+ ui.SysName = charsToString(u.Sysname[:])
+ ui.Nodename = charsToString(u.Nodename[:])
+ ui.Release = charsToString(u.Release[:])
+ ui.Version = charsToString(u.Version[:])
+ ui.Machine = charsToString(u.Machine[:])
+
+ return &ui, nil
+}
+
+func charsToString(ca []byte) string {
+ s := make([]byte, len(ca))
+ var lens int
+ for ; lens < len(ca); lens++ {
+ if ca[lens] == 0 {
+ break
+ }
+ s[lens] = ca[lens]
+ }
+ return string(s[0:lens])
+}
diff --git a/scripts/build/bash/btfgen.sh b/scripts/build/bash/btfgen.sh
new file mode 100644
index 0000000..89f6da0
--- /dev/null
+++ b/scripts/build/bash/btfgen.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+TMPDIR=$1
+ARCH=$2
+FROM=$3
+OUTPUT=$4
+BPF_SO_PATTERN="^bpf\_[a-z0-9]+\.o"
+
+echo "btfhub-archive is a big archive project, maybe take some times..."
+git clone --depth 1 https://github.com/aquasecurity/btfhub $TMPDIR/btfhub
+git clone --depth 1 https://github.com/aquasecurity/btfhub-archive/
$TMPDIR/btfhub-archive/
+mv $TMPDIR/btfhub-archive/* $TMPDIR/btfhub/archive/
+
+each_all_bpf_so_file() {
+ result=""
+ for file in `ls -a $1`
+ do
+ if [ -d $1"/"$file ]
+ then
+ if [[ $file != '.' && $file != '..' ]]
+ then
+ each_all_bpf_so_file $1"/"$file
+ fi
+ elif [[ "$file" =~ $BPF_SO_PATTERN ]]
+ then
+ echo "-o $1/$file"
+ fi
+ done
+}
+
+${TMPDIR}/btfhub/tools/btfgen.sh -a ${ARCH} $(each_all_bpf_so_file $FROM)
+mkdir -p ${OUTPUT}
+cp -r ${TMPDIR}/btfhub/custom-archive/* ${OUTPUT}
\ No newline at end of file
diff --git a/scripts/build/btf.mk b/scripts/build/btf.mk
new file mode 100644
index 0000000..b6e33cc
--- /dev/null
+++ b/scripts/build/btf.mk
@@ -0,0 +1,26 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+.PHONY: btfgen
+btfgen: generate
+ bash ${REPODIR}/scripts/build/bash/btfgen.sh /tmp x86_64 ${REPODIR}
${REPODIR}/pkg/tools/btf/files
+
+# Usually works for generate BTF file on Mac OS or windows
+.PHONY: container-btfgen
+container-btfgen: COMMAND=btfgen
+container-btfgen: container-command
\ No newline at end of file
diff --git a/scripts/release/create_bin_release.sh
b/scripts/release/create_bin_release.sh
index c6907c8..6a9401d 100644
--- a/scripts/release/create_bin_release.sh
+++ b/scripts/release/create_bin_release.sh
@@ -56,7 +56,7 @@ fi
git checkout ${TAG_NAME}
-make container-generate build
+make container-btfgen build
cd ..
