Hi Eric,
On Fri Sep 01, 2017 at 07:21:22PM +0200, Eric Auger wrote:
> SMMUV3 does not support any IOVA range TLBI command:
> SMMU_CMD_TLBI_NH_VA invalidates TLB entries by page.
> That's an issue when running DPDK on guest. DPDK uses
> hugepages but each time a hugepage is mapped on guest side,
> a storm of SMMU_CMD_TLBI_NH_VA commands get sent by the
> guest smmuv3 driver and trapped by QEMU for VFIO replay.
>
> Let's get prepared to handle implementation defined commands,
> SMMU_CMD_TLBI_NH_VA_VM, which invalidate a range of IOVAs.
>
> Upon this command, we notify the whole range in one host.
>
> Signed-off-by: Eric Auger <eric.au...@redhat.com>
> ---
> hw/arm/smmuv3-internal.h | 1 +
> hw/arm/smmuv3.c | 13 +++++++++++++
> hw/arm/trace-events | 1 +
> 3 files changed, 15 insertions(+)
>
> diff --git a/hw/arm/smmuv3-internal.h b/hw/arm/smmuv3-internal.h
> index f9f95ae..e70cf76 100644
> --- a/hw/arm/smmuv3-internal.h
> +++ b/hw/arm/smmuv3-internal.h
> @@ -289,6 +289,7 @@ enum {
> SMMU_CMD_RESUME = 0x44,
> SMMU_CMD_STALL_TERM,
> SMMU_CMD_SYNC, /* 0x46 */
> + SMMU_CMD_TLBI_NH_VA_AM = 0x8F, /* VIOMMU Impl Defined */
> };
>
> static const char *cmd_stringify[] = {
> diff --git a/hw/arm/smmuv3.c b/hw/arm/smmuv3.c
> index 9c8640f..55dc80b 100644
> --- a/hw/arm/smmuv3.c
> +++ b/hw/arm/smmuv3.c
> @@ -880,6 +880,19 @@ static int smmuv3_cmdq_consume(SMMUV3State *s)
> smmuv3_replay_iova_range(&s->smmu_state, addr, size);
> break;
> }
> + case SMMU_CMD_TLBI_NH_VA_AM:
> + {
> + int asid = extract32(cmd.word[1], 16, 16);
> + int am = extract32(cmd.word[1], 0, 16);
> + uint64_t low = extract32(cmd.word[2], 12, 20);
> + uint64_t high = cmd.word[3];
> + uint64_t addr = high << 32 | (low << 12);
> + size_t size = am << 12;
>
While testing dpdk, observed that there are map requests coming to
arm smmuv3 driver with size greater than 256M. Since the current
code supports only 256M( 16 + 12 bits), had to abuse the asid field
to pass the extra bits for address mask to get things working.
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 04e2d75..51b1d07 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -1418,7 +1418,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long
iova, size_t size,
if (smmu->options & ARM_SMMU_OPT_TLBI_ON_MAP) {
cmd.opcode = CMDQ_OP_TLBI_NH_VA_AM;
cmd.tlbi.am = size >> 12;
+ cmd.tlbi.asid = size >> 28;
granule = size;
On Qemu side,
if (cfg.disabled || cfg.bypassed) {
@@ -884,12 +899,15 @@ static int smmuv3_cmdq_consume(SMMUV3State *s)
case SMMU_CMD_TLBI_NH_VA_AM:
{
int asid = extract32(cmd.word[1], 16, 16);
int am = extract32(cmd.word[1], 0, 16);
uint64_t low = extract32(cmd.word[2], 12, 20);
uint64_t high = cmd.word[3];
uint64_t addr = high << 32 | (low << 12);
size_t size = am << 12;
+ am = am | asid << 16;
+ size = am << 12;
+
+
> + trace_smmuv3_cmdq_tlbi_nh_va_am(asid, am, addr, size);
> + smmuv3_replay_iova_range(&s->smmu_state, addr, size);
> + break;
> + }
> case SMMU_CMD_TLBI_NH_VAA:
> case SMMU_CMD_TLBI_EL3_ALL:
> case SMMU_CMD_TLBI_EL3_VA:
> diff --git a/hw/arm/trace-events b/hw/arm/trace-events
> index 15f84d6..fba33ac 100644
> --- a/hw/arm/trace-events
> +++ b/hw/arm/trace-events
> @@ -26,6 +26,7 @@ smmuv3_cmdq_opcode(const char *opcode) "<--- %s"
> smmuv3_cmdq_cfgi_ste(int streamid) " |_ streamid =%d"
> smmuv3_cmdq_cfgi_ste_range(int start, int end) " |_ start=0x%d -
> end=0x%d"
> smmuv3_cmdq_tlbi_nh_va(int asid, int vmid, uint64_t addr) " |_ asid =%d
> vmid =%d addr=0x%"PRIx64
> +smmuv3_cmdq_tlbi_nh_va_am(int asid, int am, size_t size, uint64_t addr) "
> |_ asid =%d am =%d size=0x%lx addr=0x%"PRIx64
> smmuv3_cmdq_consume_out(uint8_t prod_wrap, uint32_t prod, uint8_t cons_wrap,
> uint32_t cons) "prod_wrap:%d, prod:0x%x cons_wrap:%d cons:0x%x"
> smmuv3_update(bool is_empty, uint32_t prod, uint32_t cons, uint8_t
> prod_wrap, uint8_t cons_wrap) "q empty:%d prod:%d cons:%d p.wrap:%d p.cons:%d"
> smmuv3_update_check_cmd(int error) "cmdq not enabled or error :0x%x"
> --
> 2.5.5
>
>
--
Linu cherian
