The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6808
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
- Adds `forklimits` command for launching a process with locally changed rlimits.
- Switches qemu process to use `forklimits` with unlimited `memlock` rlimit for PCI passthrough.
From 679bad01e26cdc46c46b3f28aa3ad1e13e499f07 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parr...@canonical.com>
Date: Wed, 29 Jan 2020 16:03:29 +0000
Subject: [PATCH 1/3] lxd/main: Adds cmdGlobal.rawArgs function
Allows access to args that haven't been modified by cobra, allowing access to
"naked" flag arguments, such as "--".
rawArgs uses os.Args but only returns arguments that occur after the supplied
command name.
E.g. "lxd mycmd arg1 -- arg2" would return []string{"arg1","--","arg2"}.
Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com>
---
lxd/main.go | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/lxd/main.go b/lxd/main.go
index 1c6299e727..bdb30f38b7 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -71,6 +71,17 @@ func (c *cmdGlobal) Run(cmd *cobra.Command, args []string)
error {
return nil
}
+// rawArgs returns the raw unprocessed arguments from os.Args after the
command name arg is found.
+func (c *cmdGlobal) rawArgs(cmd *cobra.Command) []string {
+ for i, arg := range os.Args {
+ if arg == cmd.Name() && len(os.Args)-1 > i {
+ return os.Args[i+1:]
+ }
+ }
+
+ return []string{}
+}
+
func main() {
// daemon command (main)
daemonCmd := cmdDaemon{}
From 8bf0b5ae89979f11ed3bfbdc66433a046e746b30 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parr...@canonical.com>
Date: Wed, 29 Jan 2020 16:05:33 +0000
Subject: [PATCH 2/3] lxd: Adds forklimits command
The forklimits command is used to spawn processes with local rlimits specified.
You can also pass through file descriptors.
This command was added to launch qemu VM processes with increased memlock
limits.
Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com>
---
lxd/main.go | 4 ++
lxd/main_forklimits.go | 143 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 147 insertions(+)
create mode 100644 lxd/main_forklimits.go
diff --git a/lxd/main.go b/lxd/main.go
index bdb30f38b7..83a3260ea8 100644
--- a/lxd/main.go
+++ b/lxd/main.go
@@ -131,6 +131,10 @@ func main() {
forkfileCmd := cmdForkfile{global: &globalCmd}
app.AddCommand(forkfileCmd.Command())
+ // forklimits sub-command
+ forklimitsCmd := cmdForklimits{global: &globalCmd}
+ app.AddCommand(forklimitsCmd.Command())
+
// forkmigrate sub-command
forkmigrateCmd := cmdForkmigrate{global: &globalCmd}
app.AddCommand(forkmigrateCmd.Command())
diff --git a/lxd/main_forklimits.go b/lxd/main_forklimits.go
new file mode 100644
index 0000000000..6c0c7e53c1
--- /dev/null
+++ b/lxd/main_forklimits.go
@@ -0,0 +1,143 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "golang.org/x/sys/unix"
+
+ "github.com/spf13/cobra"
+)
+
+var reLimitsArg = regexp.MustCompile(`^limit=(\w+):(\w+):(\w+)$`)
+
+type cmdForklimits struct {
+ global *cmdGlobal
+}
+
+func (c *cmdForklimits) Command() *cobra.Command {
+ // Main subcommand
+ cmd := &cobra.Command{}
+ cmd.Use = "forklimits [fd=<number>...]
[limit=<name>:<softlimit>:<hardlimit>...] -- <command> <args...>"
+ cmd.Short = "Execute a task inside the container"
+ cmd.Long = `Description:
+ Execute a command with specific limits set.
+
+ This internal command is used to spawn a command with limits set. It can
also pass through one or more filed escriptors specified by fd=n arguments.
+ These are passed through in the order they are specified.
+`
+ cmd.RunE = c.Run
+ cmd.Hidden = true
+
+ return cmd
+}
+
+func (c *cmdForklimits) Run(cmd *cobra.Command, _ []string) error {
+ // Use raw args instead of cobra passed args, as we need to access the
"--" argument.
+ args := c.global.rawArgs(cmd)
+
+ if len(args) == 0 {
+ cmd.Help()
+ return nil
+ }
+
+ // Only root should run this
+ if os.Geteuid() != 0 {
+ return fmt.Errorf("This must be run as root")
+ }
+
+ type limit struct {
+ name string
+ soft string
+ hard string
+ }
+
+ var limits []limit
+ var fds []uintptr
+ var cmdParts []string
+
+ for i, arg := range args {
+ matches := reLimitsArg.FindStringSubmatch(arg)
+ if len(matches) == 4 {
+ limits = append(limits, limit{
+ name: matches[1],
+ soft: matches[2],
+ hard: matches[3],
+ })
+ } else if strings.HasPrefix(arg, "fd=") {
+ fdParts := strings.SplitN(arg, "=", 2)
+ fdNum, err := strconv.Atoi(fdParts[1])
+ if err != nil {
+ cmd.Help()
+ return fmt.Errorf("Invalid file descriptor
number")
+ }
+ fds = append(fds, uintptr(fdNum))
+ } else if arg == "--" {
+ if len(args)-1 > i {
+ cmdParts = args[i+1:]
+ }
+ break // No more passing of arguments needed.
+ } else {
+ cmd.Help()
+ return fmt.Errorf("Unrecognised argument")
+ }
+ }
+
+ // Setup rlimits.
+ for _, limit := range limits {
+ var resource int
+ var rLimit unix.Rlimit
+
+ if limit.name == "memlock" {
+ resource = unix.RLIMIT_MEMLOCK
+ } else {
+ return fmt.Errorf("Unsupported limit type: %q",
limit.name)
+ }
+
+ if limit.soft == "unlimited" {
+ rLimit.Cur = unix.RLIM_INFINITY
+ } else {
+ softLimit, err := strconv.ParseUint(limit.soft, 10, 64)
+ if err != nil {
+ return fmt.Errorf("Invalid soft limit for %q",
limit.name)
+ }
+ rLimit.Cur = softLimit
+ }
+
+ if limit.hard == "unlimited" {
+ rLimit.Max = unix.RLIM_INFINITY
+ } else {
+ hardLimit, err := strconv.ParseUint(limit.hard, 10, 64)
+ if err != nil {
+ return fmt.Errorf("Invalid hard limit for %q",
limit.name)
+ }
+ rLimit.Max = hardLimit
+ }
+
+ err := unix.Setrlimit(resource, &rLimit)
+ if err != nil {
+ return err
+ }
+ }
+
+ if len(cmdParts) == 0 {
+ cmd.Help()
+ return fmt.Errorf("Missing required command argument")
+ }
+
+ execCmd := exec.Command(cmdParts[0], cmdParts[1:]...)
+ execCmd.Stdin = os.Stdin
+ execCmd.Stdout = os.Stdout
+ execCmd.Stderr = os.Stderr
+
+ // Pass through any file descriptors specified.
+ for _, fd := range fds {
+ execCmd.ExtraFiles = append(execCmd.ExtraFiles, os.NewFile(fd,
""))
+ }
+
+ return execCmd.Run()
+}
From a03f7bc313ae2aef085456de04c1fe991937ad7e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parr...@canonical.com>
Date: Wed, 29 Jan 2020 16:06:59 +0000
Subject: [PATCH 3/3] lxd/instance/drivers/driver/qemu: Switches to launching
qemu via forklimits
This allows the memlock rlimit to be set to unlimited for PCI passthrough.
Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 25 +++++++++++++++++++------
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go
b/lxd/instance/drivers/driver_qemu.go
index 4ebdc409bf..de186751cc 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -690,7 +690,9 @@ func (vm *qemu) Start(stateful bool) error {
return err
}
- args := []string{
+ qemuCmd := []string{
+ "--",
+ qemuBinary,
"-S",
"-name", vm.Name(),
"-uuid", vmUUID,
@@ -709,7 +711,7 @@ func (vm *qemu) Start(stateful bool) error {
// Attempt to drop privileges.
if vm.state.OS.UnprivUser != "" {
- args = append(args, "-runas", vm.state.OS.UnprivUser)
+ qemuCmd = append(qemuCmd, "-runas", vm.state.OS.UnprivUser)
// Change ownership of config directory files so they are
accessible to the
// unprivileged qemu process so that the 9p share can work.
@@ -740,15 +742,26 @@ func (vm *qemu) Start(stateful bool) error {
}
if shared.IsTrue(vm.expandedConfig["limits.memory.hugepages"]) {
- args = append(args, "-mem-path", "/dev/hugepages/",
"-mem-prealloc")
+ qemuCmd = append(qemuCmd, "-mem-path", "/dev/hugepages/",
"-mem-prealloc")
}
if vm.expandedConfig["raw.qemu"] != "" {
fields := strings.Split(vm.expandedConfig["raw.qemu"], " ")
- args = append(args, fields...)
+ qemuCmd = append(qemuCmd, fields...)
}
- cmd := exec.Command(qemuBinary, args...)
+ // Run the qemu command via forklimits so we can selectively increase
ulimits.
+ forkLimitsCmd := []string{
+ "forklimits",
+ "limit=memlock:unlimited:unlimited", // Required for PCI
passthrough.
+ }
+
+ for i := range fdFiles {
+ // Pass through any file descriptors as 3+i (as first 3 file
descriptors are taken as standard).
+ forkLimitsCmd = append(forkLimitsCmd, fmt.Sprintf("fd=%d", 3+i))
+ }
+
+ cmd := exec.Command(vm.state.OS.ExecPath, append(forkLimitsCmd,
qemuCmd...)...)
var stdout bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &stdout
@@ -768,7 +781,7 @@ func (vm *qemu) Start(stateful bool) error {
err = cmd.Run()
if err != nil {
- err = errors.Wrapf(err, "Failed to run: %s %s: %s", qemuBinary,
strings.Join(args, " "), strings.TrimSpace(string(stderr.Bytes())))
+ err = errors.Wrapf(err, "Failed to run: %s: %s",
strings.Join(cmd.Args, " "), strings.TrimSpace(string(stderr.Bytes())))
op.Done(err)
return err
}
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
