Weidong,
There seems to be line wrapping issues with the patch.
Please resend it.
Thanks,
Ben
On Thu, 2008-07-03 at 17:05 +0800, Han, Weidong wrote:
> Multiple devices may under different IOMMUs, so one dmar_domain cannot
> cover this. This patch abstracts a structure kvm_vtd_domain, which let
> devices under the same IOMMU share the same dmar_domain. Then use a
> kvm_vtd_domain list to contain the all assigned devices.
>
> This patch is on top of vtd branch of amit's tree. I will rebase it if
> VT-d patches sent by Ben are merged into KVM main tree.
>
> Signed-off-by: Weidong Han <[EMAIL PROTECTED]>
> ---
> arch/x86/kvm/vtd.c | 260
> +++++++++++++++++++++++++++++---------------
> arch/x86/kvm/x86.c | 4 +-
> include/asm-x86/kvm_host.h | 14 ++-
> 3 files changed, 185 insertions(+), 93 deletions(-)
>
> diff --git a/arch/x86/kvm/vtd.c b/arch/x86/kvm/vtd.c
> index 4387c25..2365509 100644
> --- a/arch/x86/kvm/vtd.c
> +++ b/arch/x86/kvm/vtd.c
> @@ -26,29 +26,32 @@
> #include <linux/intel-iommu.h>
> #include "vtd.h"
>
> -int kvm_iommu_map_pages(struct kvm *kvm,
> - gfn_t base_gfn, unsigned long npages)
> +DEFINE_SPINLOCK(kvm_vtd_domain_lock);
> +
> +static int kvm_iommu_map_domain_pages(struct kvm *kvm,
> + struct dmar_domain *domain,
> + gfn_t base_gfn, unsigned long
> npages)
> {
> gfn_t gfn = base_gfn;
> pfn_t pfn;
> struct page *page;
> int i, rc;
>
> - if (!kvm->arch.intel_iommu_domain)
> + if (!domain)
> return -EFAULT;
>
> - printk(KERN_DEBUG "kvm_iommu_map_page: gpa = %lx\n",
> + printk(KERN_DEBUG "kvm_iommu_map_domain_page: gpa = %lx\n",
> gfn << PAGE_SHIFT);
> - printk(KERN_DEBUG "kvm_iommu_map_page: hpa = %lx\n",
> + printk(KERN_DEBUG "kvm_iommu_map_domain_page: hpa = %lx\n",
> gfn_to_pfn(kvm, base_gfn) << PAGE_SHIFT);
> - printk(KERN_DEBUG "kvm_iommu_map_page: size = %lx\n",
> + printk(KERN_DEBUG "kvm_iommu_map_domain_page: size = %lx\n",
> npages*PAGE_SIZE);
>
> for (i = 0; i < npages; i++) {
> pfn = gfn_to_pfn(kvm, gfn);
> if (pfn_valid(pfn)) {
> rc = kvm_intel_iommu_page_mapping(
> - kvm->arch.intel_iommu_domain,
> + domain,
> gfn << PAGE_SHIFT, pfn << PAGE_SHIFT,
> PAGE_SIZE, DMA_PTE_READ |
> DMA_PTE_WRITE);
> if (rc) {
> @@ -64,25 +67,146 @@ int kvm_iommu_map_pages(struct kvm *kvm,
> }
> return 0;
> }
> +
> +int kvm_iommu_map_pages(struct kvm *kvm,
> + gfn_t base_gfn, unsigned long npages)
> +{
> + struct kvm_vtd_domain *vtd_dom = NULL;
> +
> + list_for_each_entry(vtd_dom, &kvm->arch.vtd_domains, list)
> + kvm_iommu_map_domain_pages(kvm, vtd_dom->domain,
> + base_gfn, npages);
> +
> + return 0;
> +}
> EXPORT_SYMBOL_GPL(kvm_iommu_map_pages);
>
> -static int kvm_iommu_map_memslots(struct kvm *kvm)
> +static int kvm_iommu_put_domain_pages(struct kvm *kvm,
> + struct dmar_domain *domain,
> + gfn_t base_gfn, unsigned long
> npages)
> +{
> + gfn_t gfn = base_gfn;
> + struct page *page;
> + int i;
> +
> + if (!domain)
> + return -EFAULT;
> +
> + printk(KERN_DEBUG "kvm_iommu_put_domain_pages: gpa = %lx\n",
> + gfn << PAGE_SHIFT);
> + printk(KERN_DEBUG "kvm_iommu_put_domain_pages: hpa = %lx\n",
> + gfn_to_pfn(kvm, gfn) << PAGE_SHIFT);
> + printk(KERN_DEBUG "kvm_iommu_put_domain_pages: size = %lx\n",
> + npages*PAGE_SIZE);
> +
> + for (i = 0; i < npages; i++) {
> + page = gfn_to_page(kvm, gfn);
> + put_page(page);
> + gfn++;
> + }
> + return 0;
> +}
> +
> +int kvm_iommu_put_pages(struct kvm *kvm,
> + gfn_t base_gfn, unsigned long npages)
> +{
> + struct kvm_vtd_domain *vtd_dom = NULL;
> +
> + list_for_each_entry(vtd_dom, &kvm->arch.vtd_domains, list)
> + kvm_iommu_put_domain_pages(kvm, vtd_dom->domain,
> + base_gfn, npages);
> +
> + return 0;
> +}
> +EXPORT_SYMBOL_GPL(kvm_iommu_put_pages);
> +
> +static int kvm_iommu_map_domain_memslots(struct kvm *kvm,
> + struct dmar_domain *domain)
> {
> int i, rc;
> for (i = 0; i < kvm->nmemslots; i++) {
> - rc = kvm_iommu_map_pages(kvm, kvm->memslots[i].base_gfn,
> - kvm->memslots[i].npages);
> + rc = kvm_iommu_map_domain_pages(kvm, domain,
> + kvm->memslots[i].base_gfn,
> kvm->memslots[i].npages);
> if (rc)
> return rc;
> }
> return 0;
> }
>
> -static int kvm_iommu_unmap_memslots(struct kvm *kvm);
> +static int kvm_iommu_unmap_domain_memslots(struct kvm *kvm,
> + struct dmar_domain *domain)
> +{
> + int i, rc;
> + for (i = 0; i < kvm->nmemslots; i++) {
> + rc = kvm_iommu_put_domain_pages(kvm, domain,
> + kvm->memslots[i].base_gfn,
> kvm->memslots[i].npages);
> + if (rc)
> + return rc;
> + }
> + return 0;
> +}
> +
> +static struct kvm_vtd_domain *find_kvm_vtd_domain(struct kvm *kvm,
> + struct pci_dev *pdev)
> +{
> + struct kvm_vtd_domain *vtd_dom;
> + struct kvm_vtd_pt_dev_list *vtd_dev;
> +
> + list_for_each_entry(vtd_dom, &kvm->arch.vtd_domains, list) {
> + list_for_each_entry(vtd_dev, &vtd_dom->pci_pt_devices,
> list) {
> + if (vtd_dev->dev_info.busnr == pdev->bus->number
> &&
> + vtd_dev->dev_info.devfn == pdev->devfn)
> + return vtd_dom;
> + }
> + }
> +
> + return NULL;
> +}
> +
> +static struct kvm_vtd_domain *get_kvm_vtd_domain(struct kvm *kvm,
> + struct pci_dev *pdev)
> +{
> + struct kvm_vtd_domain *vtd_dom;
> +
> + vtd_dom = find_kvm_vtd_domain(kvm, pdev);
> + if (!vtd_dom) {
> + vtd_dom = kzalloc(sizeof(struct kvm_vtd_domain),
> GFP_KERNEL);
> + if (!vtd_dom)
> + return NULL;
> +
> + INIT_LIST_HEAD(&vtd_dom->pci_pt_devices);
> + if (!vtd_dom->domain) {
> + vtd_dom->domain =
> kvm_intel_iommu_domain_alloc(pdev);
> + if (!vtd_dom->domain) {
> + kfree(vtd_dom);
> + return NULL;
> + }
> +
> + if (kvm_iommu_map_domain_memslots(kvm,
> + vtd_dom->domain)) {
> + kvm_iommu_unmap_domain_memslots(kvm,
> + vtd_dom->domain);
> +
> kvm_intel_iommu_domain_exit(vtd_dom->domain);
> + kfree(vtd_dom);
> + return NULL;
> + }
> + }
> +
> + spin_lock(&kvm_vtd_domain_lock);
> + list_add(&vtd_dom->list, &kvm->arch.vtd_domains);
> + spin_unlock(&kvm_vtd_domain_lock);
> + }
> +
> + return vtd_dom;
> +}
> +
> int kvm_iommu_map_guest(struct kvm *kvm,
> - struct kvm_pci_passthrough_dev *pci_pt_dev)
> + struct kvm_pci_passthrough_dev *pci_pt_dev)
> {
> struct pci_dev *pdev = NULL;
> + struct kvm_vtd_domain *vtd_dom = NULL;
> + struct kvm_vtd_pt_dev_list *vtd_dev = NULL;
> +
>
> printk(KERN_DEBUG "kvm_iommu_map_guest: host bdf = %x:%x:%x\n",
> pci_pt_dev->host.busnr,
> @@ -94,104 +218,62 @@ int kvm_iommu_map_guest(struct kvm *kvm,
> (pdev->devfn == pci_pt_dev->host.devfn))
> goto found;
> }
> - if (kvm->arch.intel_iommu_domain) {
> -
> kvm_intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
> - kvm->arch.intel_iommu_domain = NULL;
> - }
> return -ENODEV;
> found:
> - kvm->arch.intel_iommu_domain =
> kvm_intel_iommu_domain_alloc(pdev);
> - if (kvm->arch.intel_iommu_domain == NULL)
> - printk(KERN_ERR "kvm_iommu_map_guest: domain ==
> NULL\n");
> - else
> - printk("kvm_iommu_map_guest: domain = %p\n",
> - kvm->arch.intel_iommu_domain);
> -
> - if (kvm_iommu_map_memslots(kvm)) {
> - kvm_iommu_unmap_memslots(kvm);
> + vtd_dom = get_kvm_vtd_domain(kvm, pdev);
> + if (!vtd_dom)
> return -EFAULT;
> - }
>
> - if (kvm_intel_iommu_context_mapping(
> - kvm->arch.intel_iommu_domain, pdev)) {
> - printk(KERN_ERR "Domain context map for %s failed",
> - pci_name(pdev));
> - return -EFAULT;
> - }
> - return 0;
> -}
> -EXPORT_SYMBOL_GPL(kvm_iommu_map_guest);
> + vtd_dev = kzalloc(sizeof(struct kvm_vtd_pt_dev_list),
> GFP_KERNEL);
> + if (!vtd_dev)
> + return -ENOMEM;
> + vtd_dev->dev_info.busnr = pdev->bus->number;
> + vtd_dev->dev_info.devfn = pdev->devfn;
>
> -static int kvm_iommu_put_pages(struct kvm *kvm,
> - gfn_t base_gfn, unsigned long npages)
> -{
> - gfn_t gfn = base_gfn;
> - struct page *page;
> - int i;
> + spin_lock(&kvm_vtd_domain_lock);
> + list_add(&vtd_dev->list, &vtd_dom->pci_pt_devices);
> + spin_unlock(&kvm_vtd_domain_lock);
>
> - if (!kvm->arch.intel_iommu_domain)
> + /* detach device, because it may be used by host */
> + kvm_intel_iommu_detach_dev(vtd_dom->domain,
> + pdev->bus->number, pdev->devfn);
> + if (kvm_intel_iommu_context_mapping(vtd_dom->domain, pdev)) {
> + printk(KERN_ERR "Domain context map for %s failed",
> + pci_name(pdev));
> return -EFAULT;
> -
> - printk(KERN_DEBUG "kvm_iommu_put_pages: gpa = %lx\n",
> - gfn << PAGE_SHIFT);
> - printk(KERN_DEBUG "kvm_iommu_put_pages: hpa = %lx\n",
> - gfn_to_pfn(kvm, gfn) << PAGE_SHIFT);
> - printk(KERN_DEBUG "kvm_iommu_put_pages: size = %lx\n",
> - npages*PAGE_SIZE);
> -
> - for (i = 0; i < npages; i++) {
> - page = gfn_to_page(kvm, gfn);
> - put_page(page);
> - gfn++;
> }
> - return 0;
> -}
>
> -static int kvm_iommu_unmap_memslots(struct kvm *kvm)
> -{
> - int i, rc;
> - for (i = 0; i < kvm->nmemslots; i++) {
> - rc = kvm_iommu_put_pages(kvm, kvm->memslots[i].base_gfn,
> - kvm->memslots[i].npages);
> - if (rc)
> - return rc;
> - }
> return 0;
> }
> +EXPORT_SYMBOL_GPL(kvm_iommu_map_guest);
>
> int kvm_iommu_unmap_guest(struct kvm *kvm)
> {
> - struct kvm_pci_pt_dev_list *entry;
> - struct pci_dev *pdev = NULL;
> + struct kvm_vtd_domain *vtd_dom;
> + struct kvm_vtd_pt_dev_list *vtd_dev;
> + struct list_head *pdev1, *pdev2;
>
> - if (!kvm->arch.intel_iommu_domain)
> + if (list_empty(&kvm->arch.vtd_domains))
> return 0;
>
> - list_for_each_entry(entry, &kvm->arch.pci_pt_dev_head, list) {
> - printk(KERN_DEBUG "kvm_iommu_unmap_guest: %x:%x:%x\n",
> - entry->pt_dev.host.busnr,
> - PCI_SLOT(entry->pt_dev.host.devfn),
> - PCI_FUNC(entry->pt_dev.host.devfn));
> + spin_lock(&kvm_vtd_domain_lock);
>
> - for_each_pci_dev(pdev) {
> - if ((pdev->bus->number ==
> entry->pt_dev.host.busnr) &&
> - (pdev->devfn ==
> entry->pt_dev.host.devfn))
> - goto found;
> - }
> - return -ENODEV;
> -found:
> - if (pdev == NULL) {
> - printk("kvm_iommu_unmap_guest: pdev == NULL\n");
> - return -EFAULT;
> + list_for_each_entry(vtd_dom, &kvm->arch.vtd_domains, list) {
> + list_for_each_safe(pdev1, pdev2,
> &vtd_dom->pci_pt_devices) {
> + vtd_dev = list_entry(pdev1,
> + struct kvm_vtd_pt_dev_list, list);
> + kvm_intel_iommu_detach_dev(vtd_dom->domain,
> + vtd_dev->dev_info.busnr,
> + vtd_dev->dev_info.devfn);
> + list_del(&vtd_dev->list);
> + kfree(vtd_dev);
> }
>
> - /* detach kvm dmar domain */
> - kvm_intel_iommu_detach_dev(
> - kvm->arch.intel_iommu_domain,
> - pdev->bus->number, pdev->devfn);
> + kvm_iommu_unmap_domain_memslots(kvm, vtd_dom->domain);
> + kvm_intel_iommu_domain_exit(vtd_dom->domain);
> }
> - kvm_iommu_unmap_memslots(kvm);
> - kvm_intel_iommu_domain_exit(kvm->arch.intel_iommu_domain);
> +
> + spin_unlock(&kvm_vtd_domain_lock);
> return 0;
> }
> EXPORT_SYMBOL_GPL(kvm_iommu_unmap_guest);
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 37e3a66..1bc41d9 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -386,9 +386,6 @@ static void kvm_free_pci_passthrough(struct kvm
> *kvm)
> kfree(pci_pt_dev);
> }
> write_unlock_irqrestore(&kvm_pci_pt_lock, flags);
> -
> - if (kvm_intel_iommu_found())
> - kvm->arch.intel_iommu_domain = NULL;
> }
>
> unsigned long segment_base(u16 selector)
> @@ -4251,6 +4248,7 @@ struct kvm *kvm_arch_create_vm(void)
>
> INIT_LIST_HEAD(&kvm->arch.active_mmu_pages);
> INIT_LIST_HEAD(&kvm->arch.pci_pt_dev_head);
> + INIT_LIST_HEAD(&kvm->arch.vtd_domains);
>
> return kvm;
> }
> diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
> index a2f1e3e..00b43a3 100644
> --- a/include/asm-x86/kvm_host.h
> +++ b/include/asm-x86/kvm_host.h
> @@ -343,6 +343,18 @@ struct kvm_pci_pt_dev_list {
> struct kvm_pci_passthrough_dev_kernel pt_dev;
> };
>
> +struct kvm_vtd_pt_dev_list {
> + struct list_head list;
> + struct kvm_pci_pt_info dev_info;
> +};
> +
> +/* devices under the same iommu uses the same dmar_domain */
> +struct kvm_vtd_domain {
> + struct list_head list;
> + struct list_head pci_pt_devices; /* assigned devices */
> + struct dmar_domain *domain; /* pointer to domain */
> +};
> +
> struct kvm_arch{
> int naliases;
> struct kvm_mem_alias aliases[KVM_ALIAS_SLOTS];
> @@ -356,7 +368,7 @@ struct kvm_arch{
> */
> struct list_head active_mmu_pages;
> struct list_head pci_pt_dev_head;
> - struct dmar_domain *intel_iommu_domain;
> + struct list_head vtd_domains;
> struct kvm_pic *vpic;
> struct kvm_ioapic *vioapic;
> struct kvm_pit *vpit;
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
