The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/7214
This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.
=== Description (from pull-request) ===
This PR fixes an issue introduced in https://github.com/lxc/lxd/pull/7185 that caused PCIe addressing conflicts due to qemu 9p devices assigning their own PCIe addresses that conflicted with the statically assigned NIC device addresses. This PR also now statically assigns 9p disk devices a PCIe address in a separate range to NICs, which will also be separate independent of the order the devices are configured in LXD config.
Fixes #7175
From 143ebba26f2ab6b1a2752451d5b681022fc5576e Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parr...@canonical.com>
Date: Fri, 17 Apr 2020 11:28:23 +0100
Subject: [PATCH 1/2] lxd/instance/drivers/driver/qemu/templates: Use static
PCIe address prefix for 9p devices
Uses 0x3.0x as address prefix to avoid qemu auto generating the address and
causing conflicts when ordering means they are added in between NICs being
added.
Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com>
---
lxd/instance/drivers/driver_qemu_templates.go | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/lxd/instance/drivers/driver_qemu_templates.go
b/lxd/instance/drivers/driver_qemu_templates.go
index ad65921e09..9eb9079387 100644
--- a/lxd/instance/drivers/driver_qemu_templates.go
+++ b/lxd/instance/drivers/driver_qemu_templates.go
@@ -187,6 +187,7 @@ unit = "1"
`))
// Devices use "qemu_" prefix indicating that this is a internally named
device.
+// Use 0x3.0x as the PCIe address prefix for 9p disk devices to allow up to 8
devices of this type.
var qemuDriveConfig = template.Must(template.New("qemuDriveConfig").Parse(`
# Config drive
[fsdev "qemu_config"]
@@ -199,9 +200,12 @@ path = "{{.path}}"
driver = "virtio-9p-pci"
fsdev = "qemu_config"
mount_tag = "config"
+multifunction = "on"
+addr = "0x3.0x{{.diskIndex}}"
`))
// Devices use "lxd_" prefix indicating that this is a user named device.
+// Use 0x3.0x as the PCIe address prefix for 9p disk devices to allow up to 8
devices of this type.
var qemuDriveDir = template.Must(template.New("qemuDriveDir").Parse(`
# {{.devName}} drive
[fsdev "lxd_{{.devName}}"]
@@ -220,6 +224,8 @@ sock_fd = "{{.proxyFD}}"
driver = "virtio-9p-pci"
fsdev = "lxd_{{.devName}}"
mount_tag = "{{.mountTag}}"
+multifunction = "on"
+addr = "0x3.0x{{.diskIndex}}"
`))
// Devices use "lxd_" prefix indicating that this is a user named device.
@@ -246,6 +252,7 @@ bootindex = "{{.bootIndex}}"
`))
// qemuDevTapCommon is common PCI device template for tap based netdevs.
+// Use 0x4.0x as the PCIe address prefix for nic devices to allow up to 8
devices of this type.
var qemuDevTapCommon = template.Must(template.New("qemuDevTapCommon").Parse(`
{{if ne .architecture "ppc64le" -}}
[device "qemu_pcie{{.chassisIndex}}"]
From edc887f40a818bdabd1cbf78a6cc192791efd4a3 Mon Sep 17 00:00:00 2001
From: Thomas Parrott <thomas.parr...@canonical.com>
Date: Fri, 17 Apr 2020 11:29:30 +0100
Subject: [PATCH 2/2] lxd/instance/drivers/drivers/qemu: Adds support for 9p
disk device PCIe indexes
Avoids ordering issues.
Improves comments around PCIe indexing.
Signed-off-by: Thomas Parrott <thomas.parr...@canonical.com>
---
lxd/instance/drivers/driver_qemu.go | 46 +++++++++++++++++++----------
1 file changed, 30 insertions(+), 16 deletions(-)
diff --git a/lxd/instance/drivers/driver_qemu.go
b/lxd/instance/drivers/driver_qemu.go
index 0b83a30380..38df3d9e5c 100644
--- a/lxd/instance/drivers/driver_qemu.go
+++ b/lxd/instance/drivers/driver_qemu.go
@@ -1562,12 +1562,19 @@ func (vm *qemu) generateQemuConfigFile(devConfs
[]*deviceConfig.RunConfig, fdFil
return "", err
}
- err = vm.addConfDriveConfig(sb)
+ // Indexes used for PCIe address generation (each device type group is
assigned their own PCIe address
+ // prefix in the templates). Each PCIe device is added as a
multifunction device allowing up to 8 devices
+ // of each type to be added.
+ nicIndex := 0
+ diskIndex := 0
+ chassisIndex := 5 // Internal devices defined in the templates use
indexes 1-4.
+
+ err = vm.addConfDriveConfig(sb, diskIndex)
if err != nil {
return "", err
}
+ diskIndex++ // The config drive is a 9p device which uses a PCIe
function so increment index.
- nicIndex := 0
bootIndexes, err := vm.deviceBootPriorities()
if err != nil {
return "", errors.Wrap(err, "Error calculating boot indexes")
@@ -1583,7 +1590,8 @@ func (vm *qemu) generateQemuConfigFile(devConfs
[]*deviceConfig.RunConfig, fdFil
if drive.TargetPath == "/" {
err = vm.addRootDriveConfig(sb,
bootIndexes, drive)
} else if drive.FSType == "9p" {
- err = vm.addDriveDirConfig(sb, fdFiles,
&agentMounts, drive)
+ err = vm.addDriveDirConfig(sb,
diskIndex, fdFiles, &agentMounts, drive)
+ diskIndex++ // 9p devices use a PCIe
function so increment index.
} else {
err = vm.addDriveConfig(sb,
bootIndexes, drive)
}
@@ -1595,11 +1603,14 @@ func (vm *qemu) generateQemuConfigFile(devConfs
[]*deviceConfig.RunConfig, fdFil
// Add network device.
if len(runConf.NetworkInterface) > 0 {
- err = vm.addNetDevConfig(sb, nicIndex, bootIndexes,
runConf.NetworkInterface, fdFiles)
+ err = vm.addNetDevConfig(sb, chassisIndex, nicIndex,
bootIndexes, runConf.NetworkInterface, fdFiles)
if err != nil {
return "", err
}
+
+ // NIC devices use a PCIe function so increment indexes.
nicIndex++
+ chassisIndex++
}
}
@@ -1705,10 +1716,11 @@ func (vm *qemu) addFirmwareConfig(sb *strings.Builder)
error {
}
// addConfDriveConfig adds the qemu config required for adding the config
drive.
-func (vm *qemu) addConfDriveConfig(sb *strings.Builder) error {
+func (vm *qemu) addConfDriveConfig(sb *strings.Builder, diskIndex int) error {
return qemuDriveConfig.Execute(sb, map[string]interface{}{
"architecture": vm.architectureName,
"path": filepath.Join(vm.Path(), "config"),
+ "diskIndex": diskIndex,
})
}
@@ -1754,7 +1766,7 @@ func (vm *qemu) addRootDriveConfig(sb *strings.Builder,
bootIndexes map[string]i
}
// addDriveDirConfig adds the qemu config required for adding a supplementary
drive directory share.
-func (vm *qemu) addDriveDirConfig(sb *strings.Builder, fdFiles *[]string,
agentMounts *[]instancetype.VMAgentMount, driveConf
deviceConfig.MountEntryItem) error {
+func (vm *qemu) addDriveDirConfig(sb *strings.Builder, diskIndex int, fdFiles
*[]string, agentMounts *[]instancetype.VMAgentMount, driveConf
deviceConfig.MountEntryItem) error {
mountTag := fmt.Sprintf("lxd_%s", driveConf.DevName)
agentMount := instancetype.VMAgentMount{
@@ -1775,20 +1787,22 @@ func (vm *qemu) addDriveDirConfig(sb *strings.Builder,
fdFiles *[]string, agentM
// For read only shares, do not use proxy.
if shared.StringInSlice("ro", driveConf.Opts) {
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "devName": driveConf.DevName,
- "mountTag": mountTag,
- "path": driveConf.DevPath,
- "readonly": true,
+ "devName": driveConf.DevName,
+ "mountTag": mountTag,
+ "path": driveConf.DevPath,
+ "readonly": true,
+ "diskIndex": diskIndex,
})
}
// Only use proxy for writable shares.
proxyFD := vm.addFileDescriptor(fdFiles, driveConf.DevPath)
return qemuDriveDir.Execute(sb, map[string]interface{}{
- "devName": driveConf.DevName,
- "mountTag": mountTag,
- "proxyFD": proxyFD,
- "readonly": false,
+ "devName": driveConf.DevName,
+ "mountTag": mountTag,
+ "proxyFD": proxyFD,
+ "readonly": false,
+ "diskIndex": diskIndex,
})
}
@@ -1830,7 +1844,7 @@ func (vm *qemu) addDriveConfig(sb *strings.Builder,
bootIndexes map[string]int,
}
// addNetDevConfig adds the qemu config required for adding a network device.
-func (vm *qemu) addNetDevConfig(sb *strings.Builder, nicIndex int, bootIndexes
map[string]int, nicConfig []deviceConfig.RunConfigItem, fdFiles *[]string)
error {
+func (vm *qemu) addNetDevConfig(sb *strings.Builder, chassisIndex, nicIndex
int, bootIndexes map[string]int, nicConfig []deviceConfig.RunConfigItem,
fdFiles *[]string) error {
var devName, nicName, devHwaddr, pciSlotName string
for _, nicItem := range nicConfig {
if nicItem.Key == "devName" {
@@ -1851,7 +1865,7 @@ func (vm *qemu) addNetDevConfig(sb *strings.Builder,
nicIndex int, bootIndexes m
"devHwaddr": devHwaddr,
"bootIndex": bootIndexes[devName],
"nicIndex": nicIndex,
- "chassisIndex": 5 + nicIndex,
+ "chassisIndex": chassisIndex,
}
// Detect MACVTAP interface types and figure out which tap device is
being used.
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel
