[PATCH v3 0/1] iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd()
v2 --> v3: Discard 'register' modifier for local variable 'cmd'. v1 --> v2: 1. Add patch 1, Properly handle the return value of arm_smmu_cmdq_build_cmd() 2. Remove arm_smmu_cmdq_copy_cmd(). In addition, when build command fails, out_cmd is not filled. [v2] https://lists.linuxfoundation.org/pipermail/iommu/2021-October/059831.html Zhen Lei (1): iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd() drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 11 --- 1 file changed, 8 insertions(+), 3 deletions(-) -- 2.25.1 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 1/1] iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd()
Although the parameter 'cmd' is always passed by a local array variable,
and only this function modifies it, the compiler does not know this. Every
time the 'cmd' variable is updated, a memory write operation is generated.
This generates many useless instruction operations.
To guide the compiler for proper optimization, 'cmd' is defined as a local
array variable, and copied to the output parameter at a time when the
function is returned.
The optimization effect can be viewed by running the "size arm-smmu-v3.o"
command.
Before:
textdata bss dec hex
282461332 56 2963473c2
After:
textdata bss dec hex
281341332 56 295227352
For example:
cmd[0] = FIELD_PREP(CMDQ_0_OP, ent->opcode);
case CMDQ_OP_TLBI_EL2_VA:
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
Before:
Each "cmd[0] |=" or "cmd[1] |=" operation generates a "str" instruction,
sum = 8.
ldrb w4, [x1, #8]//w4 = ent->tlbi.num
ubfiz x4, x4, #12, #5
movw0, #0x0
orrx4, x4, x3
strx4, [x2]
autiasp
ldrb w3, [x1, #9]//w3 = ent->tlbi.scale
ubfiz x3, x3, #20, #5
orrx3, x3, x4
strx3, [x2]
ldrh w4, [x1, #10] //w4 = ent->tlbi.asid
orrx3, x3, x4, lsl #48
strx3, [x2]
ldrb w3, [x1, #14] //w3 = ent->tlbi.leaf
strx3, [x2, #8]
ldrb w4, [x1, #15] //w4 = ent->tlbi.ttl
ubfiz x4, x4, #8, #2
orrx4, x4, x3
strx4, [x2, #8]
ldrb w3, [x1, #16] //ent->tlbi.tg
ubfiz x3, x3, #10, #2
orrx3, x3, x4
strx3, [x2, #8]
ldrx1, [x1, #24] //ent->tlbi.addr
andx1, x1, #0xf000
orrx1, x1, x3
strx1, [x2, #8]
ret
After:
All "cmd[0] |=" and "cmd[1] |=" operations generate a "stp" instruction,
sum = 1.
3e8:
movw0, #0x0
autiasp
stpx2, x1, [x3]
ret
btij
3fc:
ldrb w5, [x1, #8]//w5 = ent->tlbi.num
movx2, #0x22 //x2 = ent->opcode = CMDQ_0_OP
ldrb w6, [x1, #9]//w6 = ent->tlbi.scale
ubfiz x5, x5, #12, #5
ldrb w0, [x1, #16] //w0 = ent->tlbi.tg
orrx5, x5, x2
ldrb w7, [x1, #15] //w7 = ent->tlbi.ttl
ldrx4, [x1, #24] //x4 = ent->tlbi.addr
ubfiz x0, x0, #10, #2
ldrh w2, [x1, #10] //w2 = ent->tlbi.asid
ubfiz x6, x6, #20, #5
ldrb w8, [x1, #14] //w8 = ent->tlbi.leaf
andx4, x4, #0xf000
ubfiz x1, x7, #8, #2
orrx1, x0, x1
orr x2, x6, x2, lsl #48
orrx0, x4, x8
orrx2, x2, x5
orrx1, x1, x0
b 3e8
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f5848b351b19359..e55dfc14cac6005 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -234,10 +234,12 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64
*ent)
}
/* High-level queue accessors */
-static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
+static int arm_smmu_cmdq_build_cmd(u64 *out_cmd, struct arm_smmu_cmdq_ent *ent)
{
- memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
- cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
+ int i;
+ u64 cmd[CMDQ_ENT_DWORDS] = {0};
+
+ cmd[0] = FIELD_PREP(CMDQ_0_OP, ent->opcode);
switch (ent->opcode) {
case CMDQ_OP_TLBI_EL2_ALL:
@@ -332,6 +334,9 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
return -ENOENT;
}
+ for (i = 0; i < CMDQ_ENT_DWORDS; i++)
+ out_cmd[i] = cmd[i];
+
return 0;
}
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 2/2] iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd()
Although the parameter 'cmd' is always passed by a local array variable,
and only this function modifies it, the compiler does not know this. The
compiler almost always reads the value of cmd[i] from memory rather than
directly using the value cached in the register. This generates many
useless instruction operations and affects the performance to some extent.
To guide the compiler for proper optimization, 'cmd' is defined as a local
array variable, marked as register, and copied to the output parameter at
a time when the function is returned.
The optimization effect can be viewed by running the "size arm-smmu-v3.o"
command.
Before:
textdata bss dec hex
269541348 56 283586ec6
After:
textdata bss dec hex
267621348 56 281666e06
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 01e95b56ffa07d1..7cec0c967f71d86 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -234,10 +234,12 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64
*ent)
}
/* High-level queue accessors */
-static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
+static int arm_smmu_cmdq_build_cmd(u64 *out_cmd, struct arm_smmu_cmdq_ent *ent)
{
- memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
- cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
+ register u64 cmd[CMDQ_ENT_DWORDS];
+
+ cmd[0] = FIELD_PREP(CMDQ_0_OP, ent->opcode);
+ cmd[1] = 0;
switch (ent->opcode) {
case CMDQ_OP_TLBI_EL2_ALL:
@@ -332,6 +334,9 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
return -ENOENT;
}
+ out_cmd[0] = cmd[0];
+ out_cmd[1] = cmd[1];
+
return 0;
}
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 1/2] iommu/arm-smmu-v3: Properly handle the return value of arm_smmu_cmdq_build_cmd()
1. Build command CMD_SYNC cannot fail. So the return value can be ignored.
2. The arm_smmu_cmdq_build_cmd() almost never fails, the addition of
"unlikely()" can optimize the instruction pipeline.
3. Check the return value in arm_smmu_cmdq_batch_add().
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 3646bf8f021cd4c..01e95b56ffa07d1 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -409,10 +409,7 @@ static void __arm_smmu_cmdq_skip_err(struct
arm_smmu_device *smmu,
dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
/* Convert the erroneous command into a CMD_SYNC */
- if (arm_smmu_cmdq_build_cmd(cmd, _sync)) {
- dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
- return;
- }
+ arm_smmu_cmdq_build_cmd(cmd, _sync);
queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
}
@@ -860,7 +857,7 @@ static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device
*smmu,
{
u64 cmd[CMDQ_ENT_DWORDS];
- if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
+ if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
ent->opcode);
return -EINVAL;
@@ -885,11 +882,20 @@ static void arm_smmu_cmdq_batch_add(struct
arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *cmd)
{
+ int index;
+
if (cmds->num == CMDQ_BATCH_ENTRIES) {
arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmds, cmds->num, false);
cmds->num = 0;
}
- arm_smmu_cmdq_build_cmd(>cmds[cmds->num * CMDQ_ENT_DWORDS], cmd);
+
+ index = cmds->num * CMDQ_ENT_DWORDS;
+ if (unlikely(arm_smmu_cmdq_build_cmd(>cmds[index], cmd))) {
+ dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
+cmd->opcode);
+ return;
+ }
+
cmds->num++;
}
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 0/2] iommu/arm-smmu-v3: Perform some simple optimizations for arm_smmu_cmdq_build_cmd()
v1 --> v2: 1. Add patch 1, Properly handle the return value of arm_smmu_cmdq_build_cmd() 2. Remove arm_smmu_cmdq_copy_cmd(). In addition, when build command fails, out_cmd is not filled. Zhen Lei (2): iommu/arm-smmu-v3: Properly handle the return value of arm_smmu_cmdq_build_cmd() iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd() drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 29 ++--- 1 file changed, 20 insertions(+), 9 deletions(-) -- 2.26.0.106.g9fadedd ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH] iommu/arm-smmu-v3: Simplify useless instructions in arm_smmu_cmdq_build_cmd()
Although the parameter 'cmd' is always passed by a local array variable,
and only this function modifies it, the compiler does not know this. The
compiler almost always reads the value of cmd[i] from memory rather than
directly using the value cached in the register. This generates many
useless instruction operations and affects the performance to some extent.
To guide the compiler for proper optimization, 'cmd' is defined as a local
array variable, marked as register, and copied to the output parameter at
a time when the function is returned.
The optimization effect can be viewed by running the "size arm-smmu-v3.o"
command.
Before:
textdata bss dec hex
276021348 56 29006714e
After:
textdata bss dec hex
274021348 56 288067086
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 18 +++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d76bbbde558b776..50a9db5bac466c7 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -233,11 +233,19 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64
*ent)
return 0;
}
+#define arm_smmu_cmdq_copy_cmd(dst, src) \
+ do {\
+ dst[0] = src[0];\
+ dst[1] = src[1];\
+ } while (0)
+
/* High-level queue accessors */
-static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
+static int arm_smmu_cmdq_build_cmd(u64 *out_cmd, struct arm_smmu_cmdq_ent *ent)
{
- memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
- cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
+ register u64 cmd[CMDQ_ENT_DWORDS];
+
+ cmd[0] = FIELD_PREP(CMDQ_0_OP, ent->opcode);
+ cmd[1] = 0;
switch (ent->opcode) {
case CMDQ_OP_TLBI_EL2_ALL:
@@ -309,6 +317,7 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
case PRI_RESP_SUCC:
break;
default:
+ arm_smmu_cmdq_copy_cmd(out_cmd, cmd);
return -EINVAL;
}
cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
@@ -329,9 +338,12 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR,
ARM_SMMU_MEMATTR_OIWB);
break;
default:
+ arm_smmu_cmdq_copy_cmd(out_cmd, cmd);
return -ENOENT;
}
+ arm_smmu_cmdq_copy_cmd(out_cmd, cmd);
+
return 0;
}
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH] iommu/arm-smmu-v3: Stop pre-zeroing batch commands in arm_smmu_atc_inv_master()
Pre-zeroing the batched commands structure is inefficient, as individual
commands are zeroed later in arm_smmu_cmdq_build_cmd(). Therefore, only
the member 'num' needs to be initialized to 0.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 50a9db5bac466c7..e6882ae81fd08f6 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1776,10 +1776,11 @@ static int arm_smmu_atc_inv_master(struct
arm_smmu_master *master)
{
int i;
struct arm_smmu_cmdq_ent cmd;
- struct arm_smmu_cmdq_batch cmds = {};
+ struct arm_smmu_cmdq_batch cmds;
arm_smmu_atc_inv_to_cmd(0, 0, 0, );
+ cmds.num = 0;
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(master->smmu, , );
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 3/4] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_get_cmdq()
One SMMU has only one normal CMDQ. Therefore, this CMDQ is used regardless
of the core on which the command is inserted. It can be referenced
directly through "smmu->cmdq". However, one SMMU has multiple ECMDQs, and
the ECMDQ used by the core on which the command insertion is executed may
be different. So the helper function arm_smmu_get_cmdq() is added, which
returns the CMDQ/ECMDQ that the current core should use. Currently, the
code that supports ECMDQ is not added. just simply returns ">cmdq".
Many subfunctions of arm_smmu_cmdq_issue_cmdlist() use ">cmdq" or
">cmdq.q" directly. To support ECMDQ, they need to call the newly
added function arm_smmu_get_cmdq() instead.
Note that normal CMDQ is still required until ECMDQ is available.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 -
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 9be07f6915cc3c8..7814366778fda35 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -335,10 +335,14 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
return 0;
}
+static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
+{
+ return >cmdq;
+}
+
static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device
*smmu,
-u32 prod)
+struct arm_smmu_queue *q, u32 prod)
{
- struct arm_smmu_queue *q = >cmdq.q;
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_CMD_SYNC,
};
@@ -579,7 +583,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
{
unsigned long flags;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
int ret = 0;
/*
@@ -595,7 +599,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
queue_poll_init(smmu, );
do {
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
if (!queue_full(llq))
break;
@@ -614,7 +618,7 @@ static int __arm_smmu_cmdq_poll_until_msi(struct
arm_smmu_device *smmu,
{
int ret = 0;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 *cmd = (u32 *)(Q_ENT(>q, llq->prod));
queue_poll_init(smmu, );
@@ -637,12 +641,12 @@ static int __arm_smmu_cmdq_poll_until_consumed(struct
arm_smmu_device *smmu,
struct arm_smmu_ll_queue *llq)
{
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 prod = llq->prod;
int ret = 0;
queue_poll_init(smmu, );
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
do {
if (queue_consumed(llq, prod))
break;
@@ -732,7 +736,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
u32 prod;
unsigned long flags;
bool owner;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
struct arm_smmu_ll_queue llq = {
.max_n_shift = cmdq->q.llq.max_n_shift,
}, head = llq;
@@ -772,7 +776,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
if (sync) {
prod = queue_inc_prod_n(, n);
- arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
+ arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, >q, prod);
queue_write(Q_ENT(>q, prod), cmd_sync, CMDQ_ENT_DWORDS);
/*
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 1/4] iommu/arm-smmu-v3: Use command queue batching helpers to improve performance
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, use command queue batching helpers to insert multiple commands
at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 235f9bdaeaf223b..5eedb46aaceece8 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1747,15 +1747,17 @@ static int arm_smmu_atc_inv_master(struct
arm_smmu_master *master)
{
int i;
struct arm_smmu_cmdq_ent cmd;
+ struct arm_smmu_cmdq_batch cmds;
arm_smmu_atc_inv_to_cmd(0, 0, 0, );
+ cmds.num = 0;
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_issue_cmd(master->smmu, );
+ arm_smmu_cmdq_batch_add(master->smmu, , );
}
- return arm_smmu_cmdq_issue_sync(master->smmu);
+ return arm_smmu_cmdq_batch_submit(master->smmu, );
}
int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 0/4] Prepare for ECMDQ support
v1 --> v2:
1. Stop pre-zeroing batch commands. Patch 1 is modified. Other patches remain
unchanged.
Before:
struct arm_smmu_cmdq_batch cmds = {};
After:
struct arm_smmu_cmdq_batch cmds;
cmds.num = 0;
RFC --> v1
1. Resend the patches for ECMDQ preparation and remove the patches for ECMDQ
implementation.
2. Patch 2 is modified. Other patches remain unchanged.
1) Add static helper __arm_smmu_cmdq_issue_cmd(), and make
arm_smmu_cmdq_issue_cmd()
and arm_smmu_cmdq_issue_cmd_with_sync() implement based on it.
2) Remove unused arm_smmu_cmdq_issue_sync().
RFC:
https://www.spinics.net/lists/arm-kernel/msg904879.html
Zhen Lei (4):
iommu/arm-smmu-v3: Use command queue batching helpers to improve
performance
iommu/arm-smmu-v3: Add and use static helper function
arm_smmu_cmdq_issue_cmd_with_sync()
iommu/arm-smmu-v3: Add and use static helper function
arm_smmu_get_cmdq()
iommu/arm-smmu-v3: Extract reusable function
__arm_smmu_cmdq_skip_err()
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 72 -
1 file changed, 43 insertions(+), 29 deletions(-)
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 4/4] iommu/arm-smmu-v3: Extract reusable function __arm_smmu_cmdq_skip_err()
When SMMU_GERROR.CMDQP_ERR is different to SMMU_GERRORN.CMDQP_ERR, it
indicates that one or more errors have been encountered on a command queue
control page interface. We need to traverse all ECMDQs in that control
page to find all errors. For each ECMDQ error handling, it is much the
same as the CMDQ error handling. This common processing part is extracted
as a new function __arm_smmu_cmdq_skip_err().
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 7814366778fda35..f3824c37f1832a2 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -359,7 +359,8 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct
arm_smmu_device *smmu,
arm_smmu_cmdq_build_cmd(cmd, );
}
-static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
+struct arm_smmu_queue *q)
{
static const char * const cerror_str[] = {
[CMDQ_ERR_CERROR_NONE_IDX] = "No error",
@@ -370,7 +371,6 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
int i;
u64 cmd[CMDQ_ENT_DWORDS];
- struct arm_smmu_queue *q = >cmdq.q;
u32 cons = readl_relaxed(q->cons_reg);
u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
struct arm_smmu_cmdq_ent cmd_sync = {
@@ -417,6 +417,11 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
}
+static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+{
+ __arm_smmu_cmdq_skip_err(smmu, >cmdq.q);
+}
+
/*
* Command queue locking.
* This is a form of bastardised rwlock with the following major changes:
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 2/4] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_cmdq_issue_cmd_with_sync()
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, function arm_smmu_cmdq_issue_cmd_with_sync() is added to insert
the 'cmd+sync' commands at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 35 +++--
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 5eedb46aaceece8..9be07f6915cc3c8 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -845,8 +845,9 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
return ret;
}
-static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
- struct arm_smmu_cmdq_ent *ent)
+static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
+struct arm_smmu_cmdq_ent *ent,
+bool sync)
{
u64 cmd[CMDQ_ENT_DWORDS];
@@ -856,12 +857,19 @@ static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device
*smmu,
return -EINVAL;
}
- return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
+ return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);
}
-static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
+static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq_ent *ent)
{
- return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
+ return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
+}
+
+static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
+struct arm_smmu_cmdq_ent *ent)
+{
+ return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
}
static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
@@ -929,8 +937,7 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu,
u16 asid)
.tlbi.asid = asid,
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
@@ -1211,8 +1218,7 @@ static void arm_smmu_sync_ste_for_sid(struct
arm_smmu_device *smmu, u32 sid)
},
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
@@ -1825,8 +1831,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
} else {
cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
}
@@ -3340,18 +3345,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device
*smmu, bool bypass)
/* Invalidate any cached configuration */
cmd.opcode = CMDQ_OP_CFGI_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Invalidate any stale TLB entries */
if (smmu->features & ARM_SMMU_FEAT_HYP) {
cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Event queue */
writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 0/4] Prepare for ECMDQ support
RFC --> v1 1. Resend the patches for ECMDQ preparation and remove the patches for ECMDQ implementation. 2. Patch 2 is modified. Other patches remain unchanged. 1) Add static helper __arm_smmu_cmdq_issue_cmd(), and make arm_smmu_cmdq_issue_cmd() and arm_smmu_cmdq_issue_cmd_with_sync() implement based on it. 2) Remove unused arm_smmu_cmdq_issue_sync(). RFC: https://www.spinics.net/lists/arm-kernel/msg904879.html Zhen Lei (4): iommu/arm-smmu-v3: Use command queue batching helpers to improve performance iommu/arm-smmu-v3: Add and use static helper function arm_smmu_cmdq_issue_cmd_with_sync() iommu/arm-smmu-v3: Add and use static helper function arm_smmu_get_cmdq() iommu/arm-smmu-v3: Extract reusable function __arm_smmu_cmdq_skip_err() drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 71 - 1 file changed, 42 insertions(+), 29 deletions(-) -- 2.26.0.106.g9fadedd ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 3/4] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_get_cmdq()
One SMMU has only one normal CMDQ. Therefore, this CMDQ is used regardless
of the core on which the command is inserted. It can be referenced
directly through "smmu->cmdq". However, one SMMU has multiple ECMDQs, and
the ECMDQ used by the core on which the command insertion is executed may
be different. So the helper function arm_smmu_get_cmdq() is added, which
returns the CMDQ/ECMDQ that the current core should use. Currently, the
code that supports ECMDQ is not added. just simply returns ">cmdq".
Many subfunctions of arm_smmu_cmdq_issue_cmdlist() use ">cmdq" or
">cmdq.q" directly. To support ECMDQ, they need to call the newly
added function arm_smmu_get_cmdq() instead.
Note that normal CMDQ is still required until ECMDQ is available.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 -
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 282f95659580267..be2df5ad2eb51b8 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -335,10 +335,14 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
return 0;
}
+static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
+{
+ return >cmdq;
+}
+
static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device
*smmu,
-u32 prod)
+struct arm_smmu_queue *q, u32 prod)
{
- struct arm_smmu_queue *q = >cmdq.q;
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_CMD_SYNC,
};
@@ -579,7 +583,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
{
unsigned long flags;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
int ret = 0;
/*
@@ -595,7 +599,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
queue_poll_init(smmu, );
do {
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
if (!queue_full(llq))
break;
@@ -614,7 +618,7 @@ static int __arm_smmu_cmdq_poll_until_msi(struct
arm_smmu_device *smmu,
{
int ret = 0;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 *cmd = (u32 *)(Q_ENT(>q, llq->prod));
queue_poll_init(smmu, );
@@ -637,12 +641,12 @@ static int __arm_smmu_cmdq_poll_until_consumed(struct
arm_smmu_device *smmu,
struct arm_smmu_ll_queue *llq)
{
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 prod = llq->prod;
int ret = 0;
queue_poll_init(smmu, );
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
do {
if (queue_consumed(llq, prod))
break;
@@ -732,7 +736,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
u32 prod;
unsigned long flags;
bool owner;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
struct arm_smmu_ll_queue llq = {
.max_n_shift = cmdq->q.llq.max_n_shift,
}, head = llq;
@@ -772,7 +776,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
if (sync) {
prod = queue_inc_prod_n(, n);
- arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
+ arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, >q, prod);
queue_write(Q_ENT(>q, prod), cmd_sync, CMDQ_ENT_DWORDS);
/*
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 2/4] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_cmdq_issue_cmd_with_sync()
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, function arm_smmu_cmdq_issue_cmd_with_sync() is added to insert
the 'cmd+sync' commands at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 35 +++--
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index c81cd929047f573..282f95659580267 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -845,8 +845,9 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
return ret;
}
-static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
- struct arm_smmu_cmdq_ent *ent)
+static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
+struct arm_smmu_cmdq_ent *ent,
+bool sync)
{
u64 cmd[CMDQ_ENT_DWORDS];
@@ -856,12 +857,19 @@ static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device
*smmu,
return -EINVAL;
}
- return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
+ return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, sync);
}
-static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
+static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq_ent *ent)
{
- return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
+ return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
+}
+
+static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
+struct arm_smmu_cmdq_ent *ent)
+{
+ return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
}
static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
@@ -929,8 +937,7 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu,
u16 asid)
.tlbi.asid = asid,
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
@@ -1211,8 +1218,7 @@ static void arm_smmu_sync_ste_for_sid(struct
arm_smmu_device *smmu, u32 sid)
},
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
@@ -1824,8 +1830,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
} else {
cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
}
@@ -3339,18 +3344,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device
*smmu, bool bypass)
/* Invalidate any cached configuration */
cmd.opcode = CMDQ_OP_CFGI_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Invalidate any stale TLB entries */
if (smmu->features & ARM_SMMU_FEAT_HYP) {
cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Event queue */
writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/4] iommu/arm-smmu-v3: Use command queue batching helpers to improve performance
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, use command queue batching helpers to insert multiple commands
at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 235f9bdaeaf223b..c81cd929047f573 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1747,15 +1747,16 @@ static int arm_smmu_atc_inv_master(struct
arm_smmu_master *master)
{
int i;
struct arm_smmu_cmdq_ent cmd;
+ struct arm_smmu_cmdq_batch cmds = {};
arm_smmu_atc_inv_to_cmd(0, 0, 0, );
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_issue_cmd(master->smmu, );
+ arm_smmu_cmdq_batch_add(master->smmu, , );
}
- return arm_smmu_cmdq_issue_sync(master->smmu);
+ return arm_smmu_cmdq_batch_submit(master->smmu, );
}
int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 4/4] iommu/arm-smmu-v3: Extract reusable function __arm_smmu_cmdq_skip_err()
When SMMU_GERROR.CMDQP_ERR is different to SMMU_GERRORN.CMDQP_ERR, it
indicates that one or more errors have been encountered on a command queue
control page interface. We need to traverse all ECMDQs in that control
page to find all errors. For each ECMDQ error handling, it is much the
same as the CMDQ error handling. This common processing part is extracted
as a new function __arm_smmu_cmdq_skip_err().
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index be2df5ad2eb51b8..597cc0ff5ef40f0 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -359,7 +359,8 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct
arm_smmu_device *smmu,
arm_smmu_cmdq_build_cmd(cmd, );
}
-static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
+struct arm_smmu_queue *q)
{
static const char * const cerror_str[] = {
[CMDQ_ERR_CERROR_NONE_IDX] = "No error",
@@ -370,7 +371,6 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
int i;
u64 cmd[CMDQ_ENT_DWORDS];
- struct arm_smmu_queue *q = >cmdq.q;
u32 cons = readl_relaxed(q->cons_reg);
u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
struct arm_smmu_cmdq_ent cmd_sync = {
@@ -417,6 +417,11 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
}
+static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+{
+ __arm_smmu_cmdq_skip_err(smmu, >cmdq.q);
+}
+
/*
* Command queue locking.
* This is a form of bastardised rwlock with the following major changes:
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH RFC 2/8] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_cmdq_issue_cmd_with_sync()
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, function arm_smmu_cmdq_issue_cmd_with_sync() is added to insert
the 'cmd+sync' commands at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 33 +
1 file changed, 21 insertions(+), 12 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 2433d3c29b49ff2..a5361153ca1d6a4 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -858,11 +858,25 @@ static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device
*smmu,
return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
}
-static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
+static int __maybe_unused arm_smmu_cmdq_issue_sync(struct arm_smmu_device
*smmu)
{
return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
}
+static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
+struct arm_smmu_cmdq_ent *ent)
+{
+ u64 cmd[CMDQ_ENT_DWORDS];
+
+ if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
+ dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
+ent->opcode);
+ return -EINVAL;
+ }
+
+ return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, true);
+}
+
static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
struct arm_smmu_cmdq_batch *cmds,
struct arm_smmu_cmdq_ent *cmd)
@@ -928,8 +942,7 @@ void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu,
u16 asid)
.tlbi.asid = asid,
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_sync_cd(struct arm_smmu_domain *smmu_domain,
@@ -1210,8 +1223,7 @@ static void arm_smmu_sync_ste_for_sid(struct
arm_smmu_device *smmu, u32 sid)
},
};
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
@@ -1823,8 +1835,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
} else {
cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
}
@@ -3338,18 +3349,16 @@ static int arm_smmu_device_reset(struct arm_smmu_device
*smmu, bool bypass)
/* Invalidate any cached configuration */
cmd.opcode = CMDQ_OP_CFGI_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Invalidate any stale TLB entries */
if (smmu->features & ARM_SMMU_FEAT_HYP) {
cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
}
cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
- arm_smmu_cmdq_issue_cmd(smmu, );
- arm_smmu_cmdq_issue_sync(smmu);
+ arm_smmu_cmdq_issue_cmd_with_sync(smmu, );
/* Event queue */
writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH RFC 7/8] iommu/arm-smmu-v3: Add arm_smmu_ecmdq_issue_cmdlist() for non-shared ECMDQ
When a core can exclusively own an ECMDQ, competition with other cores
does not need to be considered during command insertion. Therefore, we can
delete the part of arm_smmu_cmdq_issue_cmdlist() that deals with
multi-core contention and generate a more efficient ECMDQ-specific
function arm_smmu_ecmdq_issue_cmdlist().
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 85 +
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 +
2 files changed, 86 insertions(+)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d5205030710bd1a..a088f2479fc6223 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -769,6 +769,87 @@ static void arm_smmu_cmdq_write_entries(struct
arm_smmu_cmdq *cmdq, u64 *cmds,
}
}
+/*
+ * The function is used when the current core exclusively occupies an ECMDQ.
+ * This is a reduced version of arm_smmu_cmdq_issue_cmdlist(), which eliminates
+ * a lot of unnecessary inter-core competition considerations.
+ */
+static int arm_smmu_ecmdq_issue_cmdlist(struct arm_smmu_device *smmu,
+ struct arm_smmu_cmdq *cmdq,
+ u64 *cmds, int n, bool sync)
+{
+ u32 prod;
+ unsigned long flags;
+ struct arm_smmu_ll_queue llq = {
+ .max_n_shift = cmdq->q.llq.max_n_shift,
+ }, head;
+ int ret = 0;
+
+ /* 1. Allocate some space in the queue */
+ local_irq_save(flags);
+ llq.val = READ_ONCE(cmdq->q.llq.val);
+ do {
+ u64 old;
+
+ while (!queue_has_space(, n + sync)) {
+ local_irq_restore(flags);
+ if (arm_smmu_cmdq_poll_until_not_full(smmu, ))
+ dev_err_ratelimited(smmu->dev, "ECMDQ
timeout\n");
+ local_irq_save(flags);
+ }
+
+ head.cons = llq.cons;
+ head.prod = queue_inc_prod_n(, n + sync);
+
+ old = cmpxchg_relaxed(>q.llq.val, llq.val, head.val);
+ if (old == llq.val)
+ break;
+
+ llq.val = old;
+ } while (1);
+
+ /* 2. Write our commands into the queue */
+ arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
+ if (sync) {
+ u64 cmd_sync[CMDQ_ENT_DWORDS];
+
+ prod = queue_inc_prod_n(, n);
+ arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, >q, prod);
+ queue_write(Q_ENT(>q, prod), cmd_sync, CMDQ_ENT_DWORDS);
+ }
+
+ /* 3. Ensuring commands are visible first */
+ dma_wmb();
+
+ /* 4. Advance the hardware prod pointer */
+ read_lock(>q.ecmdq_lock);
+ writel_relaxed(head.prod | cmdq->q.ecmdq_prod, cmdq->q.prod_reg);
+ read_unlock(>q.ecmdq_lock);
+
+ /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
+ if (sync) {
+ llq.prod = queue_inc_prod_n(, n);
+ ret = arm_smmu_cmdq_poll_until_sync(smmu, );
+ if (ret) {
+ dev_err_ratelimited(smmu->dev,
+ "CMD_SYNC timeout at 0x%08x [hwprod
0x%08x, hwcons 0x%08x]\n",
+ llq.prod,
+ readl_relaxed(cmdq->q.prod_reg),
+ readl_relaxed(cmdq->q.cons_reg));
+ }
+
+ /*
+* Update cmdq->q.llq.cons, to improve the success rate of
+* queue_has_space() when some new commands are inserted next
+* time.
+*/
+ WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
+ }
+
+ local_irq_restore(flags);
+ return ret;
+}
+
/*
* This is the actual insertion function, and provides the following
* ordering guarantees to callers:
@@ -798,6 +879,9 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
}, head = llq;
int ret = 0;
+ if (!cmdq->shared)
+ return arm_smmu_ecmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync);
+
/* 1. Allocate some space in the queue */
local_irq_save(flags);
llq.val = READ_ONCE(cmdq->q.llq.val);
@@ -3001,6 +3085,7 @@ static int arm_smmu_cmdq_init(struct arm_smmu_device
*smmu)
unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
atomic_long_t *bitmap;
+ cmdq->shared = 1;
atomic_set(>owner_prod, 0);
atomic_set(>lock, 0);
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 3f3a867a4626fcd..c6efbea3c0a1cda 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/a
[PATCH RFC 4/8] iommu/arm-smmu-v3: Extract reusable function __arm_smmu_cmdq_skip_err()
When SMMU_GERROR.CMDQP_ERR is different to SMMU_GERRORN.CMDQP_ERR, it
indicates that one or more errors have been encountered on a command queue
control page interface. We need to traverse all ECMDQs in that control
page to find all errors. For each ECMDQ error handling, it is much the
same as the CMDQ error handling. This common processing part is extracted
as a new function __arm_smmu_cmdq_skip_err().
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index e4af13b1e7fc015..62b2742daab3257 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -359,7 +359,8 @@ static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct
arm_smmu_device *smmu,
arm_smmu_cmdq_build_cmd(cmd, );
}
-static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+static void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
+struct arm_smmu_queue *q)
{
static const char * const cerror_str[] = {
[CMDQ_ERR_CERROR_NONE_IDX] = "No error",
@@ -370,7 +371,6 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
int i;
u64 cmd[CMDQ_ENT_DWORDS];
- struct arm_smmu_queue *q = >cmdq.q;
u32 cons = readl_relaxed(q->cons_reg);
u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
struct arm_smmu_cmdq_ent cmd_sync = {
@@ -416,6 +416,11 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
}
+static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
+{
+ __arm_smmu_cmdq_skip_err(smmu, >cmdq.q);
+}
+
/*
* Command queue locking.
* This is a form of bastardised rwlock with the following major changes:
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH RFC 5/8] iommu/arm-smmu-v3: Add support for ECMDQ register mode
Ensure that each core exclusively occupies an ECMDQ and all of them are
enabled during initialization. During this initialization process, any
errors will result in a fallback to using normal CMDQ.
When GERROR is triggered by ECMDQ, all ECMDQs need to be traversed: the
ECMDQs with errors will be processed and the ECMDQs without errors will
be skipped directly.
Compared with register SMMU_CMDQ_PROD, register SMMU_ECMDQ_PROD has one
more 'EN' bit and one more 'ERRACK' bit. Therefore, an extra member
'ecmdq_prod' is added to record the values of these two bits. Each time
register SMMU_ECMDQ_PROD is updated, the value of 'ecmdq_prod' is ORed.
After the error indicated by SMMU_GERROR.CMDQP_ERR is fixed, the 'ERRACK'
bit needs to be toggled to resume the corresponding ECMDQ. Therefore, a
rwlock is used to protect the write operation to bit 'ERRACK' during error
handling and the read operation to bit 'ERRACK' during command insertion.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 210 +++-
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 36
2 files changed, 245 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 62b2742daab3257..d7b590e911a879d 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -337,6 +337,14 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
{
+ if (smmu->ecmdq_enabled) {
+ struct arm_smmu_ecmdq *ecmdq;
+
+ ecmdq = *this_cpu_ptr(smmu->ecmdq);
+
+ return >cmdq;
+ }
+
return >cmdq;
}
@@ -421,6 +429,38 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device
*smmu)
__arm_smmu_cmdq_skip_err(smmu, >cmdq.q);
}
+static void arm_smmu_ecmdq_skip_err(struct arm_smmu_device *smmu)
+{
+ int i;
+ u32 prod, cons;
+ struct arm_smmu_queue *q;
+ struct arm_smmu_ecmdq *ecmdq;
+
+ for (i = 0; i < smmu->nr_ecmdq; i++) {
+ unsigned long flags;
+
+ ecmdq = *per_cpu_ptr(smmu->ecmdq, i);
+ q = >cmdq.q;
+
+ prod = readl_relaxed(q->prod_reg);
+ cons = readl_relaxed(q->cons_reg);
+ if (((prod ^ cons) & ECMDQ_CONS_ERR) == 0)
+ continue;
+
+ __arm_smmu_cmdq_skip_err(smmu, q);
+
+ write_lock_irqsave(>ecmdq_lock, flags);
+ q->ecmdq_prod &= ~ECMDQ_PROD_ERRACK;
+ q->ecmdq_prod |= cons & ECMDQ_CONS_ERR;
+
+ prod = readl_relaxed(q->prod_reg);
+ prod &= ~ECMDQ_PROD_ERRACK;
+ prod |= cons & ECMDQ_CONS_ERR;
+ writel(prod, q->prod_reg);
+ write_unlock_irqrestore(>ecmdq_lock, flags);
+ }
+}
+
/*
* Command queue locking.
* This is a form of bastardised rwlock with the following major changes:
@@ -817,7 +857,13 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
* d. Advance the hardware prod pointer
* Control dependency ordering from the entries becoming valid.
*/
- writel_relaxed(prod, cmdq->q.prod_reg);
+ if (smmu->ecmdq_enabled) {
+ read_lock(>q.ecmdq_lock);
+ writel_relaxed(prod | cmdq->q.ecmdq_prod,
cmdq->q.prod_reg);
+ read_unlock(>q.ecmdq_lock);
+ } else {
+ writel_relaxed(prod, cmdq->q.prod_reg);
+ }
/*
* e. Tell the next owner we're done
@@ -1675,6 +1721,9 @@ static irqreturn_t arm_smmu_gerror_handler(int irq, void
*dev)
if (active & GERROR_CMDQ_ERR)
arm_smmu_cmdq_skip_err(smmu);
+ if (active & GERROR_CMDQP_ERR)
+ arm_smmu_ecmdq_skip_err(smmu);
+
writel(gerror, smmu->base + ARM_SMMU_GERRORN);
return IRQ_HANDLED;
}
@@ -2941,6 +2990,20 @@ static int arm_smmu_cmdq_init(struct arm_smmu_device
*smmu)
return ret;
}
+static int arm_smmu_ecmdq_init(struct arm_smmu_cmdq *cmdq)
+{
+ unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
+
+ atomic_set(>owner_prod, 0);
+ atomic_set(>lock, 0);
+
+ cmdq->valid_map = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
+ if (!cmdq->valid_map)
+ return -ENOMEM;
+
+ return 0;
+}
+
static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
{
int ret;
@@ -3304,6 +3367,7 @@ static int arm_smmu_device_disable(struct arm_smmu_device
*smmu)
static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
{
+
[PATCH RFC 8/8] iommu/arm-smmu-v3: Add support for less than one ECMDQ per core
Due to limited hardware resources, the number of ECMDQs may be less than
the number of cores. If the number of ECMDQs is greater than the number of
numa nodes, ensure that each node has at least one ECMDQ. This is because
ECMDQ queue memory is requested from the NUMA node where it resides, which
may result in better command filling and insertion performance.
The current ECMDQ implementation reuses the command insertion function
arm_smmu_cmdq_issue_cmdlist() of the normal CMDQ. This function already
supports multiple cores concurrent insertion commands.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 101 ++--
1 file changed, 92 insertions(+), 9 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index a088f2479fc6223..55f651ce42e7a51 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3636,14 +3636,15 @@ static int arm_smmu_device_reset(struct arm_smmu_device
*smmu, bool bypass)
static int arm_smmu_ecmdq_layout(struct arm_smmu_device *smmu)
{
- int cpu;
- struct arm_smmu_ecmdq *ecmdq;
+ int cpu, node, nr_remain, nr_nodes = 0;
+ int *nr_ecmdqs;
+ struct arm_smmu_ecmdq *ecmdq, **ecmdqs;
- if (num_possible_cpus() <= smmu->nr_ecmdq) {
- ecmdq = devm_alloc_percpu(smmu->dev, *ecmdq);
- if (!ecmdq)
- return -ENOMEM;
+ ecmdq = devm_alloc_percpu(smmu->dev, *ecmdq);
+ if (!ecmdq)
+ return -ENOMEM;
+ if (num_possible_cpus() <= smmu->nr_ecmdq) {
for_each_possible_cpu(cpu)
*per_cpu_ptr(smmu->ecmdq, cpu) = per_cpu_ptr(ecmdq,
cpu);
@@ -3653,7 +3654,79 @@ static int arm_smmu_ecmdq_layout(struct arm_smmu_device
*smmu)
return 0;
}
- return -ENOSPC;
+ for_each_node(node)
+ if (nr_cpus_node(node))
+ nr_nodes++;
+
+ if (nr_nodes >= smmu->nr_ecmdq) {
+ dev_err(smmu->dev, "%d ECMDQs is less than %d nodes\n",
smmu->nr_ecmdq, nr_nodes);
+ return -ENOSPC;
+ }
+
+ nr_ecmdqs = kcalloc(MAX_NUMNODES, sizeof(int), GFP_KERNEL);
+ if (!nr_ecmdqs)
+ return -ENOMEM;
+
+ ecmdqs = kcalloc(smmu->nr_ecmdq, sizeof(*ecmdqs), GFP_KERNEL);
+ if (!ecmdqs) {
+ kfree(nr_ecmdqs);
+ return -ENOMEM;
+ }
+
+ /* [1] Ensure that each node has at least one ECMDQ */
+ nr_remain = smmu->nr_ecmdq - nr_nodes;
+ for_each_node(node) {
+ /*
+* Calculate the number of ECMDQs to be allocated to this node.
+* NR_ECMDQS_PER_CPU = nr_remain / num_possible_cpus();
+* When nr_cpus_node(node) is not zero, less than one ECMDQ
+* may be left due to truncation rounding.
+*/
+ nr_ecmdqs[node] = nr_cpus_node(node) * nr_remain /
num_possible_cpus();
+ nr_remain -= nr_ecmdqs[node];
+ }
+
+ /* Divide the remaining ECMDQs */
+ while (nr_remain) {
+ for_each_node(node) {
+ if (!nr_remain)
+ break;
+
+ if (nr_ecmdqs[node] >= nr_cpus_node(node))
+ continue;
+
+ nr_ecmdqs[node]++;
+ nr_remain--;
+ }
+ }
+
+ for_each_node(node) {
+ int i, round, shared = 0;
+
+ if (!nr_cpus_node(node))
+ continue;
+
+ /* An ECMDQ has been reserved for each node at above [1] */
+ nr_ecmdqs[node]++;
+
+ if (nr_ecmdqs[node] < nr_cpus_node(node))
+ shared = 1;
+
+ i = 0;
+ for_each_cpu(cpu, cpumask_of_node(node)) {
+ round = i % nr_ecmdqs[node];
+ if (i++ < nr_ecmdqs[node]) {
+ ecmdqs[round] = per_cpu_ptr(ecmdq, cpu);
+ ecmdqs[round]->cmdq.shared = shared;
+ }
+ *per_cpu_ptr(smmu->ecmdq, cpu) = ecmdqs[round];
+ }
+ }
+
+ kfree(nr_ecmdqs);
+ kfree(ecmdqs);
+
+ return 0;
}
static int arm_smmu_ecmdq_probe(struct arm_smmu_device *smmu)
@@ -3718,10 +3791,20 @@ static int arm_smmu_ecmdq_probe(struct arm_smmu_device
*smmu)
struct arm_smmu_queue *q;
ecmdq = *per_cpu_ptr(smmu->ecmdq, cpu);
- ecmdq->base = cp_base + addr;
-
q = >cmdq.q;
+ /*
+* The boot option "maxcpus=" can limit the number of online
+* CPUs. The CPUs that are not sel
[PATCH RFC 3/8] iommu/arm-smmu-v3: Add and use static helper function arm_smmu_get_cmdq()
One SMMU has only one normal CMDQ. Therefore, this CMDQ is used regardless
of the core on which the command is inserted. It can be referenced
directly through "smmu->cmdq". However, one SMMU has multiple ECMDQs, and
the ECMDQ used by the core on which the command insertion is executed may
be different. So the helper function arm_smmu_get_cmdq() is added, which
returns the CMDQ/ECMDQ that the current core should use. Currently, the
code that supports ECMDQ is not added. just simply returns ">cmdq".
Many subfunctions of arm_smmu_cmdq_issue_cmdlist() use ">cmdq" or
">cmdq.q" directly. To support ECMDQ, they need to call the newly
added function arm_smmu_get_cmdq() instead.
Note that normal CMDQ is still required until ECMDQ is available.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 22 -
1 file changed, 13 insertions(+), 9 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index a5361153ca1d6a4..e4af13b1e7fc015 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -335,10 +335,14 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct
arm_smmu_cmdq_ent *ent)
return 0;
}
+static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu)
+{
+ return >cmdq;
+}
+
static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device
*smmu,
-u32 prod)
+struct arm_smmu_queue *q, u32 prod)
{
- struct arm_smmu_queue *q = >cmdq.q;
struct arm_smmu_cmdq_ent ent = {
.opcode = CMDQ_OP_CMD_SYNC,
};
@@ -578,7 +582,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
{
unsigned long flags;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
int ret = 0;
/*
@@ -594,7 +598,7 @@ static int arm_smmu_cmdq_poll_until_not_full(struct
arm_smmu_device *smmu,
queue_poll_init(smmu, );
do {
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
if (!queue_full(llq))
break;
@@ -613,7 +617,7 @@ static int __arm_smmu_cmdq_poll_until_msi(struct
arm_smmu_device *smmu,
{
int ret = 0;
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 *cmd = (u32 *)(Q_ENT(>q, llq->prod));
queue_poll_init(smmu, );
@@ -636,12 +640,12 @@ static int __arm_smmu_cmdq_poll_until_consumed(struct
arm_smmu_device *smmu,
struct arm_smmu_ll_queue *llq)
{
struct arm_smmu_queue_poll qp;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
u32 prod = llq->prod;
int ret = 0;
queue_poll_init(smmu, );
- llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
+ llq->val = READ_ONCE(cmdq->q.llq.val);
do {
if (queue_consumed(llq, prod))
break;
@@ -731,7 +735,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
u32 prod;
unsigned long flags;
bool owner;
- struct arm_smmu_cmdq *cmdq = >cmdq;
+ struct arm_smmu_cmdq *cmdq = arm_smmu_get_cmdq(smmu);
struct arm_smmu_ll_queue llq = {
.max_n_shift = cmdq->q.llq.max_n_shift,
}, head = llq;
@@ -771,7 +775,7 @@ static int arm_smmu_cmdq_issue_cmdlist(struct
arm_smmu_device *smmu,
arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
if (sync) {
prod = queue_inc_prod_n(, n);
- arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
+ arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, >q, prod);
queue_write(Q_ENT(>q, prod), cmd_sync, CMDQ_ENT_DWORDS);
/*
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH RFC 6/8] iommu/arm-smmu-v3: Ensure that a set of associated commands are inserted in the same ECMDQ
The SYNC command only ensures that the command that precedes it in the
same ECMDQ must be executed, but cannot synchronize the commands in other
ECMDQs. If an unmap involves multiple commands, some commands are executed
on one core, and the other commands are executed on another core. In this
case, after the SYNC execution is complete, the execution of all preceded
commands can not be ensured.
Prevent the process that performs a set of associated commands insertion
from being migrated to other cores ensures that all commands are inserted
into the same ECMDQ.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 40 +
1 file changed, 33 insertions(+), 7 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d7b590e911a879d..d5205030710bd1a 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -233,6 +233,18 @@ static int queue_remove_raw(struct arm_smmu_queue *q, u64
*ent)
return 0;
}
+static void arm_smmu_preempt_disable(struct arm_smmu_device *smmu)
+{
+ if (smmu->ecmdq_enabled)
+ preempt_disable();
+}
+
+static void arm_smmu_preempt_enable(struct arm_smmu_device *smmu)
+{
+ if (smmu->ecmdq_enabled)
+ preempt_enable();
+}
+
/* High-level queue accessors */
static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
{
@@ -1016,6 +1028,7 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain
*smmu_domain,
},
};
+ arm_smmu_preempt_disable(smmu);
spin_lock_irqsave(_domain->devices_lock, flags);
list_for_each_entry(master, _domain->devices, domain_head) {
for (i = 0; i < master->num_streams; i++) {
@@ -1026,6 +1039,7 @@ static void arm_smmu_sync_cd(struct arm_smmu_domain
*smmu_domain,
spin_unlock_irqrestore(_domain->devices_lock, flags);
arm_smmu_cmdq_batch_submit(smmu, );
+ arm_smmu_preempt_enable(smmu);
}
static int arm_smmu_alloc_cd_leaf_table(struct arm_smmu_device *smmu,
@@ -1814,30 +1828,36 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova,
size_t size,
static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
{
- int i;
+ int i, ret;
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_cmdq_batch cmds = {};
+ struct arm_smmu_device *smmu = master->smmu;
arm_smmu_atc_inv_to_cmd(0, 0, 0, );
+ arm_smmu_preempt_disable(smmu);
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_batch_add(master->smmu, , );
+ arm_smmu_cmdq_batch_add(smmu, , );
}
- return arm_smmu_cmdq_batch_submit(master->smmu, );
+ ret = arm_smmu_cmdq_batch_submit(smmu, );
+ arm_smmu_preempt_enable(smmu);
+
+ return ret;
}
int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
unsigned long iova, size_t size)
{
- int i;
+ int i, ret;
unsigned long flags;
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_master *master;
struct arm_smmu_cmdq_batch cmds = {};
+ struct arm_smmu_device *smmu = smmu_domain->smmu;
- if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
+ if (!(smmu->features & ARM_SMMU_FEAT_ATS))
return 0;
/*
@@ -1859,6 +1879,7 @@ int arm_smmu_atc_inv_domain(struct arm_smmu_domain
*smmu_domain, int ssid,
arm_smmu_atc_inv_to_cmd(ssid, iova, size, );
+ arm_smmu_preempt_disable(smmu);
spin_lock_irqsave(_domain->devices_lock, flags);
list_for_each_entry(master, _domain->devices, domain_head) {
if (!master->ats_enabled)
@@ -1866,12 +1887,15 @@ int arm_smmu_atc_inv_domain(struct arm_smmu_domain
*smmu_domain, int ssid,
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_batch_add(smmu_domain->smmu, , );
+ arm_smmu_cmdq_batch_add(smmu, , );
}
}
spin_unlock_irqrestore(_domain->devices_lock, flags);
- return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, );
+ ret = arm_smmu_cmdq_batch_submit(smmu, );
+ arm_smmu_preempt_enable(smmu);
+
+ return ret;
}
/* IO_PGTABLE API */
@@ -1924,6 +1948,7 @@ static void __arm_smmu_tlb_inv_range(struct
arm_smmu_cmdq_ent *cmd,
num_pages = size >> tg;
}
+ arm_smmu_preempt_disable(smmu);
while (iova < end) {
if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
/*
@@ -1955,6 +1980,7 @@ static void __arm_smmu_tlb_inv_range(struct
arm_sm
[PATCH RFC 1/8] iommu/arm-smmu-v3: Use command queue batching helpers to improve performance
The obvious key to the performance optimization of commit 587e6c10a7ce
("iommu/arm-smmu-v3: Reduce contention during command-queue insertion") is
to allow multiple cores to insert commands in parallel after a brief mutex
contention.
Obviously, inserting as many commands at a time as possible can reduce the
number of times the mutex contention participates, thereby improving the
overall performance. At least it reduces the number of calls to function
arm_smmu_cmdq_issue_cmdlist().
Therefore, use command queue batching helpers to insert multiple commands
at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index dd20b01771c4bd2..2433d3c29b49ff2 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1746,15 +1746,16 @@ static int arm_smmu_atc_inv_master(struct
arm_smmu_master *master)
{
int i;
struct arm_smmu_cmdq_ent cmd;
+ struct arm_smmu_cmdq_batch cmds = {};
arm_smmu_atc_inv_to_cmd(0, 0, 0, );
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
- arm_smmu_cmdq_issue_cmd(master->smmu, );
+ arm_smmu_cmdq_batch_add(master->smmu, , );
}
- return arm_smmu_cmdq_issue_sync(master->smmu);
+ return arm_smmu_cmdq_batch_submit(master->smmu, );
}
int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH RFC 0/8] iommu/arm-smmu-v3: add support for ECMDQ register mode
SMMU v3.3 added a new feature, which is Enhanced Command queue interface for reducing contention when submitting Commands to the SMMU, in this patch set, ECMDQ is the abbreviation of Enhanced Command Queue. When the hardware supports ECMDQ and each core can exclusively use one ECMDQ, each core does not need to compete with other cores when using its own ECMDQ. This means that each core can insert commands in parallel. If each ECMDQ can execute commands in parallel, the overall performance may be better. However, our hardware currently does not support multiple ECMDQ execute commands in parallel. In order to reuse existing code, I originally still call arm_smmu_cmdq_issue_cmdlist() to insert commands. Even so, however, there was a performance improvement of nearly 12% in strict mode. The test environment is the EMU, which simulates the connection of the 200 Gbit/s NIC. Number of queues:passthrough lazy strict(ECMDQ) strict(CMDQ) 6 188180 162 145--> 11.7% improvement 8 188188 184 183--> 0.55% improvement In recent days, I implemented a new function without competition with other cores to replace arm_smmu_cmdq_issue_cmdlist() when a core can have an ECMDQ. I'm guessing it might get better performance results. Because the EMU is too slow, it will take a while before the relevant data is available. Zhen Lei (8): iommu/arm-smmu-v3: Use command queue batching helpers to improve performance iommu/arm-smmu-v3: Add and use static helper function arm_smmu_cmdq_issue_cmd_with_sync() iommu/arm-smmu-v3: Add and use static helper function arm_smmu_get_cmdq() iommu/arm-smmu-v3: Extract reusable function __arm_smmu_cmdq_skip_err() iommu/arm-smmu-v3: Add support for ECMDQ register mode iommu/arm-smmu-v3: Ensure that a set of associated commands are inserted in the same ECMDQ iommu/arm-smmu-v3: Add arm_smmu_ecmdq_issue_cmdlist() for non-shared ECMDQ iommu/arm-smmu-v3: Add support for less than one ECMDQ per core drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 483 ++-- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 37 ++ 2 files changed, 489 insertions(+), 31 deletions(-) -- 2.26.0.106.g9fadedd ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu-v3: remove unnecessary oom message
Fixes scripts/checkpatch.pl warning:
WARNING: Possible unnecessary 'out of memory' message
Remove it can help us save a bit of memory.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 2ddc3cd5a7d1..fd7c55b44881 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -2787,10 +2787,8 @@ static int arm_smmu_init_l1_strtab(struct
arm_smmu_device *smmu)
void *strtab = smmu->strtab_cfg.strtab;
cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
- if (!cfg->l1_desc) {
- dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
+ if (!cfg->l1_desc)
return -ENOMEM;
- }
for (i = 0; i < cfg->num_l1_ents; ++i) {
arm_smmu_write_strtab_l1_desc(strtab, >l1_desc[i]);
@@ -3581,10 +3579,8 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
bool bypass;
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
- if (!smmu) {
- dev_err(dev, "failed to allocate arm_smmu_device\n");
+ if (!smmu)
return -ENOMEM;
- }
smmu->dev = dev;
if (dev->of_node) {
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu: remove unnecessary oom message
Fixes scripts/checkpatch.pl warning:
WARNING: Possible unnecessary 'out of memory' message
Remove it can help us save a bit of memory.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu/arm-smmu.c | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c
b/drivers/iommu/arm/arm-smmu/arm-smmu.c
index dba15f312cbd..db7b1e101bc5 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
@@ -2051,10 +2051,8 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
irqreturn_t (*global_fault)(int irq, void *dev);
smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
- if (!smmu) {
- dev_err(dev, "failed to allocate arm_smmu_device\n");
+ if (!smmu)
return -ENOMEM;
- }
smmu->dev = dev;
if (dev->of_node)
@@ -2095,10 +2093,8 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
GFP_KERNEL);
- if (!smmu->irqs) {
- dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
+ if (!smmu->irqs)
return -ENOMEM;
- }
for (i = 0; i < num_irqs; ++i) {
int irq = platform_get_irq(pdev, i);
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/vt-d: remove unnecessary oom message
Fixes scripts/checkpatch.pl warning:
WARNING: Possible unnecessary 'out of memory' message
Remove it can help us save a bit of memory.
Signed-off-by: Zhen Lei
---
drivers/iommu/intel/dmar.c | 2 --
drivers/iommu/intel/iommu.c | 6 +-
2 files changed, 1 insertion(+), 7 deletions(-)
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index 84057cb9596c..ede7525b4ec3 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -148,8 +148,6 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned
long event)
} else {
info = kzalloc(size, GFP_KERNEL);
if (!info) {
- pr_warn("Out of memory when allocating notify_info "
- "for %s.\n", pci_name(dev));
if (dmar_dev_scope_status == 0)
dmar_dev_scope_status = -ENOMEM;
return NULL;
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index be35284a2016..432f4019d1af 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1780,11 +1780,8 @@ static int iommu_init_domains(struct intel_iommu *iommu)
spin_lock_init(>lock);
iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
- if (!iommu->domain_ids) {
- pr_err("%s: Allocating domain id array failed\n",
- iommu->name);
+ if (!iommu->domain_ids)
return -ENOMEM;
- }
size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);
iommu->domains = kzalloc(size, GFP_KERNEL);
@@ -3220,7 +3217,6 @@ static int __init init_dmars(void)
g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
GFP_KERNEL);
if (!g_iommus) {
- pr_err("Allocating global iommu array failed\n");
ret = -ENOMEM;
goto error;
}
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/ipmmu-vmsa: remove unnecessary oom message
Fixes scripts/checkpatch.pl warning:
WARNING: Possible unnecessary 'out of memory' message
Remove it can help us save a bit of memory.
Signed-off-by: Zhen Lei
---
drivers/iommu/ipmmu-vmsa.c | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 51ea6f00db2f..d31e3890f5e2 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -992,10 +992,8 @@ static int ipmmu_probe(struct platform_device *pdev)
int ret;
mmu = devm_kzalloc(>dev, sizeof(*mmu), GFP_KERNEL);
- if (!mmu) {
- dev_err(>dev, "cannot allocate device data\n");
+ if (!mmu)
return -ENOMEM;
- }
mmu->dev = >dev;
spin_lock_init(>lock);
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 0/2] iommu: Fix spelling mistakes
v2 --> v3: 1) Add some new fixes for the latest linux-next: drivers/iommu/fsl_pamu_domain.c:366: Traverese ==> Traverse drivers/iommu/mtk_iommu.c:977: Uppon ==> Upon drivers/iommu/intel/svm.c:488: shuld ==> should drivers/iommu/intel/svm.c:920: requeset ==> request drivers/iommu/intel/dmar.c:2131: Specifiction ==> Specification 2) Add a new fix "Additionally, The ==> Additionally, the", discovered by Will Deacon 3) Add some new fixes for the header files of iommu/iova The header files to be checked are as follows: include/linux/iommu*.h include/linux/iova.h include/uapi/linux/iommu.h 4) Changes to files "iova.h" and "iova.c" are grouped into a new patch. v1 --> v2: 1. Merge into one patch 2. Add a new fix "appropriatley --> appropriately" in iommu.c, discovered by John Garry Zhen Lei (2): iommu/iova: Fix spelling mistakes iommu: Fix spelling mistakes drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/init.c | 2 +- drivers/iommu/amd/iommu.c | 2 +- drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +- drivers/iommu/fsl_pamu.c | 2 +- drivers/iommu/fsl_pamu_domain.c | 2 +- drivers/iommu/intel/dmar.c| 8 drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/irq_remapping.c | 2 +- drivers/iommu/intel/svm.c | 4 ++-- drivers/iommu/iommu.c | 6 +++--- drivers/iommu/iova.c | 2 +- drivers/iommu/mtk_iommu.c | 4 ++-- drivers/iommu/omap-iommu.c| 2 +- drivers/iommu/sun50i-iommu.c | 2 +- include/linux/iova.h | 10 +- 16 files changed, 27 insertions(+), 27 deletions(-) -- 2.25.1 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 1/2] iommu/iova: Fix spelling mistakes
Fix some spelling mistakes in comments found by "codespell":
detroyed ==> destroyed
defered ==> deferred
entrie ==> entry
alloced ==> allocated
regularily ==> regularly
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 2 +-
include/linux/iova.h | 10 +-
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index b6cf5f16123b..cce4571548c4 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -594,7 +594,7 @@ static void fq_destroy_all_entries(struct iova_domain
*iovad)
int cpu;
/*
-* This code runs when the iova_domain is being detroyed, so don't
+* This code runs when the iova_domain is being destroyed, so don't
* bother to free iovas, just call the entry_dtor on all remaining
* entries.
*/
diff --git a/include/linux/iova.h b/include/linux/iova.h
index 71d8a2de6635..16f671b04a37 100644
--- a/include/linux/iova.h
+++ b/include/linux/iova.h
@@ -49,12 +49,12 @@ typedef void (* iova_entry_dtor)(unsigned long data);
/* Timeout (in ms) after which entries are flushed from the Flush-Queue */
#define IOVA_FQ_TIMEOUT10
-/* Flush Queue entry for defered flushing */
+/* Flush Queue entry for deferred flushing */
struct iova_fq_entry {
unsigned long iova_pfn;
unsigned long pages;
unsigned long data;
- u64 counter; /* Flush counter when this entrie was added */
+ u64 counter; /* Flush counter when this entry was added */
};
/* Per-CPU Flush Queue structure */
@@ -68,8 +68,8 @@ struct iova_fq {
struct iova_domain {
spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */
struct rb_root rbroot; /* iova domain rbtree root */
- struct rb_node *cached_node; /* Save last alloced node */
- struct rb_node *cached32_node; /* Save last 32-bit alloced node */
+ struct rb_node *cached_node; /* Save last allocated node */
+ struct rb_node *cached32_node; /* Save last 32-bit allocated node */
unsigned long granule;/* pfn granularity for this domain */
unsigned long start_pfn; /* Lower limit for this domain */
unsigned long dma_32bit_pfn;
@@ -91,7 +91,7 @@ struct iova_domain {
iova_entry_dtor entry_dtor; /* IOMMU driver specific destructor for
iova entry */
- struct timer_list fq_timer; /* Timer to regularily empty the
+ struct timer_list fq_timer; /* Timer to regularly empty the
flush-queues */
atomic_t fq_timer_on; /* 1 when timer is active, 0
when not */
--
2.25.1
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 2/2] iommu: Fix spelling mistakes
Fix some spelling mistakes in comments found by "codespell":
alignement ==> alignment
implemtation ==> implementation
assignement ==> assignment
initally ==> initially
Returs ==> Returns
Traverese ==> Traverse
guarentees ==> guarantees
resgister ==> register
insufficent ==> insufficient
Specifiction ==> Specification
creats ==> creates
tabke ==> table
shuld ==> should
requeset ==> request
funcions ==> functions
distiguish ==> distinguish
phyiscal ==> physical
Uppon ==> Upon
consits ==> consists
And two were discovered manually by John Garry and Will Deacon:
appropriatley ==> appropriately
Additionally, The ==> Additionally, the
Signed-off-by: Zhen Lei
---
drivers/iommu/amd/amd_iommu_types.h | 2 +-
drivers/iommu/amd/init.c | 2 +-
drivers/iommu/amd/iommu.c | 2 +-
drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +-
drivers/iommu/fsl_pamu.c | 2 +-
drivers/iommu/fsl_pamu_domain.c | 2 +-
drivers/iommu/intel/dmar.c| 8
drivers/iommu/intel/iommu.c | 2 +-
drivers/iommu/intel/irq_remapping.c | 2 +-
drivers/iommu/intel/svm.c | 4 ++--
drivers/iommu/iommu.c | 6 +++---
drivers/iommu/mtk_iommu.c | 4 ++--
drivers/iommu/omap-iommu.c| 2 +-
drivers/iommu/sun50i-iommu.c | 2 +-
14 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu_types.h
b/drivers/iommu/amd/amd_iommu_types.h
index 94c1a7a9876d..67a6c2fb4de9 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -446,7 +446,7 @@ extern struct irq_remap_table **irq_lookup_table;
/* Interrupt remapping feature used? */
extern bool amd_iommu_irq_remap;
-/* kmem_cache to get tables with 128 byte alignement */
+/* kmem_cache to get tables with 128 byte alignment */
extern struct kmem_cache *amd_iommu_irq_cache;
/*
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 4e4fb0f4e412..52d450962288 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -2040,7 +2040,7 @@ static int intcapxt_irqdomain_activate(struct irq_domain
*domain,
xt.destid_24_31 = cfg->dest_apicid >> 24;
/**
-* Current IOMMU implemtation uses the same IRQ for all
+* Current IOMMU implementation uses the same IRQ for all
* 3 IOMMU interrupts.
*/
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index b1fbf2c83df5..1b635d4c2142 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1809,7 +1809,7 @@ int __init amd_iommu_init_api(void)
* The following functions belong to the exported interface of AMD IOMMU
*
* This interface allows access to lower level functions of the IOMMU
- * like protection domain handling and assignement of devices to domains
+ * like protection domain handling and assignment of devices to domains
* which is not possible with the dma_ops interface.
*
*/
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c
b/drivers/iommu/arm/arm-smmu/arm-smmu.c
index dba15f312cbd..79db6e8c5e31 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
@@ -1360,7 +1360,7 @@ static struct iommu_device *arm_smmu_probe_device(struct
device *dev)
ret = arm_smmu_register_legacy_master(dev, );
/*
-* If dev->iommu_fwspec is initally NULL,
arm_smmu_register_legacy_master()
+* If dev->iommu_fwspec is initially NULL,
arm_smmu_register_legacy_master()
* will allocate/initialise a new one. Thus we need to update
fwspec for
* later use.
*/
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index fc38b1fba7cf..53aff2766367 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -246,7 +246,7 @@ void get_ome_index(u32 *omi_index, struct device *dev)
* @stash_dest_hint: L1, L2 or L3
* @vcpu: vpcu target for a particular cache type.
*
- * Returs stash on success or ~(u32)0 on failure.
+ * Returns stash on success or ~(u32)0 on failure.
*
*/
u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index a47f47307109..2da312645279 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -363,7 +363,7 @@ static struct iommu_group
*get_shared_pci_device_group(struct pci_dev *pdev)
struct pci_bus *bus = pdev->bus;
/*
-* Traverese the pci bus device list to get
+* Traverse the pci bus device list to get
* the shared iommu group.
*/
while (bus) {
diff --git a/drivers/iommu/intel/d
[PATCH 1/1] dma-mapping: remove trailing spaces and tabs
Run the following command to find and remove the trailing spaces and tabs: find kernel/dma/ -type f | xargs sed -r -i 's/[ \t]+$//' Signed-off-by: Zhen Lei --- kernel/dma/coherent.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/dma/coherent.c b/kernel/dma/coherent.c index 5b5b6c7ec7f2..794e76b03b34 100644 --- a/kernel/dma/coherent.c +++ b/kernel/dma/coherent.c @@ -111,7 +111,7 @@ static int dma_assign_coherent_memory(struct device *dev, * Declare a region of memory to be handed out by dma_alloc_coherent() when it * is asked for coherent memory for this device. This shall only be used * from platform code, usually based on the device tree description. - * + * * phys_addr is the CPU physical address to which the memory is currently * assigned (this will be ioremapped so the CPU can access the region). * -- 2.25.1 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu-v3: Decrease the queue size of evtq and priq
Commit d25f6ead162e ("iommu/arm-smmu-v3: Increase maximum size of queues")
expands the cmdq queue size to improve the success rate of concurrent
command queue space allocation by multiple cores. However, this extension
does not apply to evtq and priq, because for both of them, the SMMU driver
is the consumer. Instead, memory resources are wasted. Therefore, the
queue size of evtq and priq is restored to the original setting, one page.
Fixes: d25f6ead162e ("iommu/arm-smmu-v3: Increase maximum size of queues")
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 46e8c49214a872e..8e5094fa2863df5 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -184,6 +184,7 @@
#else
#define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
#endif
+#define Q_MIN_SZ_SHIFT (PAGE_SHIFT)
/*
* Stream table.
@@ -366,14 +367,14 @@
/* Event queue */
#define EVTQ_ENT_SZ_SHIFT 5
#define EVTQ_ENT_DWORDS((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
-#define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
+#define EVTQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
#define EVTQ_0_ID GENMASK_ULL(7, 0)
/* PRI queue */
#define PRIQ_ENT_SZ_SHIFT 4
#define PRIQ_ENT_DWORDS((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
-#define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
+#define PRIQ_MAX_SZ_SHIFT (Q_MIN_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
#define PRIQ_0_SID GENMASK_ULL(31, 0)
#define PRIQ_0_SSIDGENMASK_ULL(51, 32)
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu: Delete a duplicate check in iommu_change_dev_def_domain()
Function iommu_group_store_type() is the only caller of the static
function iommu_change_dev_def_domain() and has performed
"if (WARN_ON(!group))" detection before calling it. So the one here is
redundant.
Signed-off-by: Zhen Lei
---
drivers/iommu/iommu.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 971068da67cb91d..8cdf6a1c4bfd773 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -3059,9 +3059,6 @@ static int iommu_change_dev_def_domain(struct iommu_group
*group,
int ret, dev_def_dom;
struct device *dev;
- if (!group)
- return -EINVAL;
-
mutex_lock(>mutex);
if (group->default_domain != group->domain) {
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 0/1] iommu: Clear a lot of spelling mistakes
v1 --> v2: 1. Merge into one patch 2. Add a new fix "appropriatley --> appropriately" in iommu.c, discovered by John Garry Zhen Lei (1): iommu: Clear a lot of spelling mistakes drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/init.c | 2 +- drivers/iommu/amd/iommu.c | 2 +- drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +- drivers/iommu/fsl_pamu.c | 2 +- drivers/iommu/intel/dmar.c| 6 +++--- drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/irq_remapping.c | 2 +- drivers/iommu/iommu.c | 6 +++--- drivers/iommu/iova.c | 2 +- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/omap-iommu.c| 2 +- drivers/iommu/sun50i-iommu.c | 2 +- 13 files changed, 17 insertions(+), 17 deletions(-) -- 2.26.0.106.g9fadedd ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 1/1] iommu: Clear a lot of spelling mistakes
All spelling mistakes are in the comments, no functional change.
Signed-off-by: Zhen Lei
---
drivers/iommu/amd/amd_iommu_types.h | 2 +-
drivers/iommu/amd/init.c | 2 +-
drivers/iommu/amd/iommu.c | 2 +-
drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +-
drivers/iommu/fsl_pamu.c | 2 +-
drivers/iommu/intel/dmar.c| 6 +++---
drivers/iommu/intel/iommu.c | 2 +-
drivers/iommu/intel/irq_remapping.c | 2 +-
drivers/iommu/iommu.c | 6 +++---
drivers/iommu/iova.c | 2 +-
drivers/iommu/mtk_iommu.c | 2 +-
drivers/iommu/omap-iommu.c| 2 +-
drivers/iommu/sun50i-iommu.c | 2 +-
13 files changed, 17 insertions(+), 17 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu_types.h
b/drivers/iommu/amd/amd_iommu_types.h
index 94c1a7a9876d554..67a6c2fb4de9e2a 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -446,7 +446,7 @@ extern struct irq_remap_table **irq_lookup_table;
/* Interrupt remapping feature used? */
extern bool amd_iommu_irq_remap;
-/* kmem_cache to get tables with 128 byte alignement */
+/* kmem_cache to get tables with 128 byte alignment */
extern struct kmem_cache *amd_iommu_irq_cache;
/*
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index d006724f4dc2122..d749837dcecc875 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -2040,7 +2040,7 @@ static int intcapxt_irqdomain_activate(struct irq_domain
*domain,
xt.destid_24_31 = cfg->dest_apicid >> 24;
/**
-* Current IOMMU implemtation uses the same IRQ for all
+* Current IOMMU implementation uses the same IRQ for all
* 3 IOMMU interrupts.
*/
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 80e8e1916dd17c9..fa2c98857f3a7d0 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1812,7 +1812,7 @@ int __init amd_iommu_init_dma_ops(void)
* The following functions belong to the exported interface of AMD IOMMU
*
* This interface allows access to lower level functions of the IOMMU
- * like protection domain handling and assignement of devices to domains
+ * like protection domain handling and assignment of devices to domains
* which is not possible with the dma_ops interface.
*
*/
diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c
b/drivers/iommu/arm/arm-smmu/arm-smmu.c
index 6f72c4d208cad84..a56e0d91b5a70a2 100644
--- a/drivers/iommu/arm/arm-smmu/arm-smmu.c
+++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c
@@ -1361,7 +1361,7 @@ static struct iommu_device *arm_smmu_probe_device(struct
device *dev)
ret = arm_smmu_register_legacy_master(dev, );
/*
-* If dev->iommu_fwspec is initally NULL,
arm_smmu_register_legacy_master()
+* If dev->iommu_fwspec is initially NULL,
arm_smmu_register_legacy_master()
* will allocate/initialise a new one. Thus we need to update
fwspec for
* later use.
*/
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index fc38b1fba7cff0a..53aff27663673a0 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -246,7 +246,7 @@ void get_ome_index(u32 *omi_index, struct device *dev)
* @stash_dest_hint: L1, L2 or L3
* @vcpu: vpcu target for a particular cache type.
*
- * Returs stash on success or ~(u32)0 on failure.
+ * Returns stash on success or ~(u32)0 on failure.
*
*/
u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index 1757ac1e1623e9a..2a2d176b36ec0f2 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -45,7 +45,7 @@ struct dmar_res_callback {
/*
* Assumptions:
- * 1) The hotplug framework guarentees that DMAR unit will be hot-added
+ * 1) The hotplug framework guarantees that DMAR unit will be hot-added
*before IO devices managed by that unit.
* 2) The hotplug framework guarantees that DMAR unit will be hot-removed
*after IO devices managed by that unit.
@@ -960,10 +960,10 @@ static void unmap_iommu(struct intel_iommu *iommu)
/**
* map_iommu: map the iommu's registers
* @iommu: the iommu to map
- * @phys_addr: the physical address of the base resgister
+ * @phys_addr: the physical address of the base register
*
* Memory map the iommu's registers. Start w/ a single page, and
- * possibly expand if that turns out to be insufficent.
+ * possibly expand if that turns out to be insufficient.
*/
static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
{
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 708f430af1c4403..ba5b0137b4b1fb4 100644
--- a/drivers/i
[PATCH 7/8] iommu/arm-smmu: Fix spelling mistake "initally" -> "initially"
There is a spelling mistake in a comment, fix it. Signed-off-by: Zhen Lei --- drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index d8c6bfde6a61587..8e4e8fea106b612 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -1358,7 +1358,7 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) ret = arm_smmu_register_legacy_master(dev, ); /* -* If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() +* If dev->iommu_fwspec is initially NULL, arm_smmu_register_legacy_master() * will allocate/initialise a new one. Thus we need to update fwspec for * later use. */ -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 8/8] iommu/vt-d: fix a couple of spelling mistakes
There are several spelling mistakes, as follows:
guarentees ==> guarantees
resgister ==> register
insufficent ==> insufficient
creats ==> creates
tabke ==> take
Signed-off-by: Zhen Lei
---
drivers/iommu/intel/dmar.c | 6 +++---
drivers/iommu/intel/iommu.c | 2 +-
drivers/iommu/intel/irq_remapping.c | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c
index d5c51b5c20aff4b..bb6f0880f6f4db0 100644
--- a/drivers/iommu/intel/dmar.c
+++ b/drivers/iommu/intel/dmar.c
@@ -45,7 +45,7 @@ struct dmar_res_callback {
/*
* Assumptions:
- * 1) The hotplug framework guarentees that DMAR unit will be hot-added
+ * 1) The hotplug framework guarantees that DMAR unit will be hot-added
*before IO devices managed by that unit.
* 2) The hotplug framework guarantees that DMAR unit will be hot-removed
*after IO devices managed by that unit.
@@ -960,10 +960,10 @@ static void unmap_iommu(struct intel_iommu *iommu)
/**
* map_iommu: map the iommu's registers
* @iommu: the iommu to map
- * @phys_addr: the physical address of the base resgister
+ * @phys_addr: the physical address of the base register
*
* Memory map the iommu's registers. Start w/ a single page, and
- * possibly expand if that turns out to be insufficent.
+ * possibly expand if that turns out to be insufficient.
*/
static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
{
diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index ee0932307d646bb..f9a2277fba99f9f 100644
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -288,7 +288,7 @@ static inline void context_clear_entry(struct context_entry
*context)
/*
* This domain is a statically identity mapping domain.
- * 1. This domain creats a static 1:1 mapping to all usable memory.
+ * 1. This domain creates a static 1:1 mapping to all usable memory.
* 2. It maps to each iommu if successful.
* 3. Each iommu mapps to this domain if successful.
*/
diff --git a/drivers/iommu/intel/irq_remapping.c
b/drivers/iommu/intel/irq_remapping.c
index 611ef5243cb63b9..12e9f2cf84e5101 100644
--- a/drivers/iommu/intel/irq_remapping.c
+++ b/drivers/iommu/intel/irq_remapping.c
@@ -74,7 +74,7 @@ struct intel_ir_data {
* ->iommu->register_lock
* Note:
* intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called
- * in single-threaded environment with interrupt disabled, so no need to tabke
+ * in single-threaded environment with interrupt disabled, so no need to take
* the dmar_global_lock.
*/
DEFINE_RAW_SPINLOCK(irq_2_ir_lock);
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 6/8] iommu/amd: fix a couple of spelling mistakes
There are several spelling mistakes, as follows:
alignement ==> alignment
programing ==> programming
implemtation ==> implementation
assignement ==> assignment
By the way, both "programing" and "programming" are acceptable, but the
latter seems more formal.
Signed-off-by: Zhen Lei
---
drivers/iommu/amd/amd_iommu_types.h | 2 +-
drivers/iommu/amd/init.c| 4 ++--
drivers/iommu/amd/iommu.c | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu_types.h
b/drivers/iommu/amd/amd_iommu_types.h
index 6937e3674a16e26..dc1814c355cff77 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -446,7 +446,7 @@ struct irq_remap_table {
/* Interrupt remapping feature used? */
extern bool amd_iommu_irq_remap;
-/* kmem_cache to get tables with 128 byte alignement */
+/* kmem_cache to get tables with 128 byte alignment */
extern struct kmem_cache *amd_iommu_irq_cache;
/*
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 321f5906e6ed3a5..48799002b3571d1 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -1734,7 +1734,7 @@ static void __init init_iommu_perf_ctr(struct amd_iommu
*iommu)
goto pc_false;
/*
-* Disable power gating by programing the performance counter
+* Disable power gating by programming the performance counter
* source to 20 (i.e. counts the reads and writes from/to IOMMU
* Reserved Register [MMIO Offset 1FF8h] that are ignored.),
* which never get incremented during this init phase.
@@ -2088,7 +2088,7 @@ static int intcapxt_irqdomain_activate(struct irq_domain
*domain,
xt.destid_24_31 = cfg->dest_apicid >> 24;
/**
-* Current IOMMU implemtation uses the same IRQ for all
+* Current IOMMU implementation uses the same IRQ for all
* 3 IOMMU interrupts.
*/
writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index a69a8b573e40d00..d14e4698f507b89 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -1865,7 +1865,7 @@ int __init amd_iommu_init_dma_ops(void)
* The following functions belong to the exported interface of AMD IOMMU
*
* This interface allows access to lower level functions of the IOMMU
- * like protection domain handling and assignement of devices to domains
+ * like protection domain handling and assignment of devices to domains
* which is not possible with the dma_ops interface.
*
*/
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 2/8] iommu/omap: Fix spelling mistake "alignement" -> "alignment"
There is a spelling mistake in a comment, fix it.
Signed-off-by: Zhen Lei
---
drivers/iommu/omap-iommu.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 71f29c0927fc710..b2a6ab700ec43d1 100644
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -1754,7 +1754,7 @@ static int __init omap_iommu_init(void)
{
struct kmem_cache *p;
const slab_flags_t flags = SLAB_HWCACHE_ALIGN;
- size_t align = 1 << 10; /* L2 pagetable alignement */
+ size_t align = 1 << 10; /* L2 pagetable alignment */
struct device_node *np;
int ret;
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 3/8] iommu/mediatek: Fix spelling mistake "phyiscal" -> "physical"
There is a spelling mistake in a comment, fix it.
Signed-off-by: Zhen Lei
---
drivers/iommu/mtk_iommu.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 6ecc007f07cd52e..c8c9bf1d70b29dc 100644
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -160,7 +160,7 @@ struct mtk_iommu_domain {
* The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the
* bit32 of the CPU physical address always is needed to set, and for Region
* 'E', the CPU physical address keep as is.
- * Additionally, The iommu consumers always use the CPU phyiscal address.
+ * Additionally, The iommu consumers always use the CPU physical address.
*/
#define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x14000UL
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 4/8] iommu/sun50i: Fix spelling mistake "consits" -> "consists"
There is a spelling mistake in a comment, fix it. Signed-off-by: Zhen Lei --- drivers/iommu/sun50i-iommu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index ea6db1341916524..7685b96b2d445a7 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -149,7 +149,7 @@ static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value) * 4096 4-bytes Directory Table Entries (DTE), each pointing to a Page * Table (PT). * - * Each PT consits of 256 4-bytes Page Table Entries (PTE), each + * Each PT consists of 256 4-bytes Page Table Entries (PTE), each * pointing to a 4kB page of physical memory. * * The IOMMU supports a single DT, pointed by the IOMMU_TTB_REG -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 5/8] iommu: fix a couple of spelling mistakes
There are several spelling mistakes, as follows: funcions ==> functions distiguish ==> distinguish detroyed ==> destroyed Signed-off-by: Zhen Lei --- drivers/iommu/iommu.c | 4 ++-- drivers/iommu/iova.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d0b0a15dba8413c..0f4e9a6122ee58f 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1453,7 +1453,7 @@ struct iommu_group *pci_device_group(struct device *dev) /* * Look for existing groups on non-isolated functions on the same -* slot and aliases of those funcions, if any. No need to clear +* slot and aliases of those functions, if any. No need to clear * the search bitmap, the tested devfns are still valid. */ group = get_pci_function_alias_group(pdev, (unsigned long *)devfns); @@ -2267,7 +2267,7 @@ struct iommu_domain *iommu_get_dma_domain(struct device *dev) * iterating over the devices in a group. Ideally we'd have a single * device which represents the requestor ID of the group, but we also * allow IOMMU drivers to create policy defined minimum sets, where - * the physical hardware may be able to distiguish members, but we + * the physical hardware may be able to distinguish members, but we * wish to group them at a higher level (ex. untrusted multi-function * PCI devices). Thus we attach each device. */ diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index e6e2fa85271c3f8..bf710b0a3713e21 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -524,7 +524,7 @@ static void fq_destroy_all_entries(struct iova_domain *iovad) int cpu; /* -* This code runs when the iova_domain is being detroyed, so don't +* This code runs when the iova_domain is being destroyed, so don't * bother to free iovas, just call the entry_dtor on all remaining * entries. */ -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/8] iommu/pamu: fix a couple of spelling mistakes
There are several spelling mistakes, as follows:
Returs ==> Returns
defaul ==> default
assocaited ==> associated
Signed-off-by: Zhen Lei
---
drivers/iommu/fsl_pamu.c| 2 +-
drivers/iommu/fsl_pamu_domain.c | 2 +-
drivers/iommu/fsl_pamu_domain.h | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c
index b9a974d9783113d..48ebbf0daa21cf9 100644
--- a/drivers/iommu/fsl_pamu.c
+++ b/drivers/iommu/fsl_pamu.c
@@ -503,7 +503,7 @@ void get_ome_index(u32 *omi_index, struct device *dev)
* @stash_dest_hint: L1, L2 or L3
* @vcpu: vpcu target for a particular cache type.
*
- * Returs stash on success or ~(u32)0 on failure.
+ * Returns stash on success or ~(u32)0 on failure.
*
*/
u32 get_stash_id(u32 stash_dest_hint, u32 vcpu)
diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c
index b2110767caf49c8..be664cd18c51970 100644
--- a/drivers/iommu/fsl_pamu_domain.c
+++ b/drivers/iommu/fsl_pamu_domain.c
@@ -418,7 +418,7 @@ static struct iommu_domain *fsl_pamu_domain_alloc(unsigned
type)
pr_debug("dma_domain allocation failed\n");
return NULL;
}
- /* defaul geometry 64 GB i.e. maximum system address */
+ /* default geometry 64 GB i.e. maximum system address */
dma_domain->iommu_domain. geometry.aperture_start = 0;
dma_domain->iommu_domain.geometry.aperture_end = (1ULL << 36) - 1;
dma_domain->iommu_domain.geometry.force_aperture = true;
diff --git a/drivers/iommu/fsl_pamu_domain.h b/drivers/iommu/fsl_pamu_domain.h
index 2865d42782e8021..4f508fa041080e3 100644
--- a/drivers/iommu/fsl_pamu_domain.h
+++ b/drivers/iommu/fsl_pamu_domain.h
@@ -24,7 +24,7 @@ struct fsl_dma_domain {
*/
dma_addr_t geom_size;
/*
-* Number of windows assocaited with this domain.
+* Number of windows associated with this domain.
* During domain initialization, it is set to the
* the maximum number of subwindows allowed for a LIODN.
* Minimum value for this is 1 indicating a single PAMU
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 0/8] iommu: fix a couple of spelling mistakes detected by codespell tool
This detection and correction covers the entire driver/iommu directory. Zhen Lei (8): iommu/pamu: fix a couple of spelling mistakes iommu/omap: Fix spelling mistake "alignement" -> "alignment" iommu/mediatek: Fix spelling mistake "phyiscal" -> "physical" iommu/sun50i: Fix spelling mistake "consits" -> "consists" iommu: fix a couple of spelling mistakes iommu/amd: fix a couple of spelling mistakes iommu/arm-smmu: Fix spelling mistake "initally" -> "initially" iommu/vt-d: fix a couple of spelling mistakes drivers/iommu/amd/amd_iommu_types.h | 2 +- drivers/iommu/amd/init.c | 4 ++-- drivers/iommu/amd/iommu.c | 2 +- drivers/iommu/arm/arm-smmu/arm-smmu.c | 2 +- drivers/iommu/fsl_pamu.c | 2 +- drivers/iommu/fsl_pamu_domain.c | 2 +- drivers/iommu/fsl_pamu_domain.h | 2 +- drivers/iommu/intel/dmar.c| 6 +++--- drivers/iommu/intel/iommu.c | 2 +- drivers/iommu/intel/irq_remapping.c | 2 +- drivers/iommu/iommu.c | 4 ++-- drivers/iommu/iova.c | 2 +- drivers/iommu/mtk_iommu.c | 2 +- drivers/iommu/omap-iommu.c| 2 +- drivers/iommu/sun50i-iommu.c | 2 +- 15 files changed, 19 insertions(+), 19 deletions(-) -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu-v3: add bit field SFM into GERROR_ERR_MASK
In arm_smmu_gerror_handler(), the value of the SMMU_GERROR register is
filtered by GERROR_ERR_MASK. However, the GERROR_ERR_MASK does not contain
the SFM bit. As a result, the subsequent error processing is not performed
when only the SFM error occurs.
Fixes: 48ec83bcbcf5 ("iommu/arm-smmu: Add initial driver support for ARM SMMUv3
devices")
Reported-by: Rui Zhu
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index f985817c967a257..230b6f6b39016cd 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -115,7 +115,7 @@
#define GERROR_PRIQ_ABT_ERR(1 << 3)
#define GERROR_EVTQ_ABT_ERR(1 << 2)
#define GERROR_CMDQ_ERR(1 << 0)
-#define GERROR_ERR_MASK0xfd
+#define GERROR_ERR_MASK0x1fd
#define ARM_SMMU_GERRORN 0x64
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v5 0/1] perf/smmuv3: Don't reserve the PMCG register spaces
v4 --> v5:
1. Give up doing the mapping for the entire SMMU register space.
2. Fix some compile warnings. Sorry. So sorry.
v3 --> v4:
1. Delete the unnecessary encapsulation function
smmu_pmu_get_and_ioremap_resource().
2. Discard adding MODULE_SOFTDEP.
v2 --> v3:
Patch 3 is updated because https://lkml.org/lkml/2021/1/22/532 has been queued
in advance.
v1 --> v2:
According to Robin Murphy's suggestion: https://lkml.org/lkml/2021/1/20/470
Don't reserve the PMCG register spaces, and reserve the entire SMMU register
space.
v1:
Since the PMCG may implement its resigters space(4KB Page0 and 4KB Page1)
within the SMMUv3 64KB Page0. In this case, when the SMMUv3 driver reserves the
64KB Page0 resource in advance, the PMCG driver try to reserve its Page0 and
Page1 resources, a resource conflict occurs.
commit 52f3fab0067d6fa ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") reduce the resource reservation range of the SMMUv3
driver, it only reserves the first 0xe00 bytes in the 64KB Page0, to avoid
the above-mentioned resource conflicts.
But the SMMUv3.3 add support for ECMDQ, its registers space is also implemented
in the SMMUv3 64KB Page0. This means we need to build two separate mappings.
New features may be added in the future, and more independent mappings may be
required. The simple problem is complicated because the user expects to map the
entire SMMUv3 64KB Page0.
Therefore, the proper solution is: If the PMCG register resources are located in
the 64KB Page0 of the SMMU, the PMCG driver does not reserve the conflict
resources
when the SMMUv3 driver has reserved the conflict resources before. Instead, the
PMCG
driver only performs devm_ioremap() to ensure that it can work properly.
Zhen Lei (1):
perf/smmuv3: Don't reserve the PMCG register spaces
drivers/perf/arm_smmuv3_pmu.c | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v5 1/1] perf/smmuv3: Don't reserve the PMCG register spaces
According to the SMMUv3 specification:
Each PMCG counter group is represented by one 4KB page (Page 0) with one
optional additional 4KB page (Page 1), both of which are at IMPLEMENTATION
DEFINED base addresses.
This means that the PMCG register spaces may be within the 64KB pages of
the SMMUv3 register space. When both the SMMU and PMCG drivers reserve
their own resources, a resource conflict occurs.
To avoid this conflict, don't reserve the PMCG regions.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
Reviewed-by: Robin Murphy
---
drivers/perf/arm_smmuv3_pmu.c | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 74474bb322c3f26..8f0b71b5d08a815 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -793,17 +793,30 @@ static int smmu_pmu_probe(struct platform_device *pdev)
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
- smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0,
_0);
- if (IS_ERR(smmu_pmu->reg_base))
- return PTR_ERR(smmu_pmu->reg_base);
+ /*
+* The register spaces of the PMCG may be in the register space of
+* other devices. For example, SMMU. Therefore, the PMCG resources are
+* not reserved to avoid resource conflicts with other drivers.
+*/
+ res_0 = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res_0)
+ return -EINVAL;
+ smmu_pmu->reg_base = devm_ioremap(dev, res_0->start,
resource_size(res_0));
+ if (!smmu_pmu->reg_base)
+ return -ENOMEM;
cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR);
/* Determine if page 1 is present */
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) {
- smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);
- if (IS_ERR(smmu_pmu->reloc_base))
- return PTR_ERR(smmu_pmu->reloc_base);
+ struct resource *res_1;
+
+ res_1 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (!res_1)
+ return -EINVAL;
+ smmu_pmu->reloc_base = devm_ioremap(dev, res_1->start,
resource_size(res_1));
+ if (!smmu_pmu->reloc_base)
+ return -ENOMEM;
} else {
smmu_pmu->reloc_base = smmu_pmu->reg_base;
}
--
2.26.0.106.g9fadedd
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v4 0/2] perf/smmuv3: Don't reserve the PMCG register spaces
v3 --> v4:
1. Delete the unnecessary encapsulation function
smmu_pmu_get_and_ioremap_resource().
2. Discard adding MODULE_SOFTDEP.
v2 --> v3:
Patch 3 is updated because https://lkml.org/lkml/2021/1/22/532 has been queued
in advance.
v1 --> v2:
According to Robin Murphy's suggestion: https://lkml.org/lkml/2021/1/20/470
Don't reserve the PMCG register spaces, and reserve the entire SMMU register
space.
v1:
Since the PMCG may implement its resigters space(4KB Page0 and 4KB Page1)
within the SMMUv3 64KB Page0. In this case, when the SMMUv3 driver reserves the
64KB Page0 resource in advance, the PMCG driver try to reserve its Page0 and
Page1 resources, a resource conflict occurs.
commit 52f3fab0067d6fa ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") reduce the resource reservation range of the SMMUv3
driver, it only reserves the first 0xe00 bytes in the 64KB Page0, to avoid
the above-mentioned resource conflicts.
But the SMMUv3.3 add support for ECMDQ, its registers space is also implemented
in the SMMUv3 64KB Page0. This means we need to build two separate mappings.
New features may be added in the future, and more independent mappings may be
required. The simple problem is complicated because the user expects to map the
entire SMMUv3 64KB Page0.
Therefore, the proper solution is: If the PMCG register resources are located in
the 64KB Page0 of the SMMU, the PMCG driver does not reserve the conflict
resources
when the SMMUv3 driver has reserved the conflict resources before. Instead, the
PMCG
driver only performs devm_ioremap() to ensure that it can work properly.
Zhen Lei (2):
perf/smmuv3: Don't reserve the PMCG register spaces
iommu/arm-smmu-v3: Reserving the entire SMMU register space
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
drivers/perf/arm_smmuv3_pmu.c | 25 +++--
3 files changed, 23 insertions(+), 28 deletions(-)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v4 2/2] iommu/arm-smmu-v3: Reserving the entire SMMU register space
commit 52f3fab0067d ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") only reserves the basic SMMU register space. So
the ECMDQ register space is not covered, it should be mapped again. Due
to the size of this ECMDQ resource is not fixed, depending on
SMMU_IDR6.CMDQ_CONTROL_PAGE_LOG2NUMQ. Processing its resource reservation
to avoid resource conflict with PMCG is a bit more complicated.
Therefore, the resources of the PMCG are not reserved, and the entire SMMU
resources are reserved.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
2 files changed, 4 insertions(+), 22 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f04c55a7503c790..fde24403b06a9e3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3476,14 +3476,6 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
return err;
}
-static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t
start,
- resource_size_t size)
-{
- struct resource res = DEFINE_RES_MEM(start, size);
-
- return devm_ioremap_resource(dev, );
-}
-
static int arm_smmu_device_probe(struct platform_device *pdev)
{
int irq, ret;
@@ -3519,22 +3511,14 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
}
ioaddr = res->start;
- /*
-* Don't map the IMPLEMENTATION DEFINED regions, since they may contain
-* the PMCG registers which are reserved by the PMU driver.
-*/
- smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
+ smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
- if (arm_smmu_resource_size(smmu) > SZ_64K) {
- smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
- ARM_SMMU_REG_SZ);
- if (IS_ERR(smmu->page1))
- return PTR_ERR(smmu->page1);
- } else {
+ if (arm_smmu_resource_size(smmu) > SZ_64K)
+ smmu->page1 = smmu->base + SZ_64K;
+ else
smmu->page1 = smmu->base;
- }
/* Interrupt lines */
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index da525f46dab4fc1..63f2b476987d6ae 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -152,8 +152,6 @@
#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
-#define ARM_SMMU_REG_SZ0xe00
-
/* Common MSI config fields */
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
#define MSI_CFG2_SHGENMASK(5, 4)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v4 1/2] perf/smmuv3: Don't reserve the PMCG register spaces
According to the SMMUv3 specification:
Each PMCG counter group is represented by one 4KB page (Page 0) with one
optional additional 4KB page (Page 1), both of which are at IMPLEMENTATION
DEFINED base addresses.
This means that the PMCG register spaces may be within the 64KB pages of
the SMMUv3 register space. When both the SMMU and PMCG drivers reserve
their own resources, a resource conflict occurs.
To avoid this conflict, don't reserve the PMCG regions.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 25 +++--
1 file changed, 19 insertions(+), 6 deletions(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 74474bb322c3f26..5e894f957c7b935 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -793,17 +793,30 @@ static int smmu_pmu_probe(struct platform_device *pdev)
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
- smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0,
_0);
- if (IS_ERR(smmu_pmu->reg_base))
- return PTR_ERR(smmu_pmu->reg_base);
+ /*
+* The register spaces of the PMCG may be in the register space of
+* other devices. For example, SMMU. Therefore, the PMCG resources are
+* not reserved to avoid resource conflicts with other drivers.
+*/
+ res_0 = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res_0)
+ return ERR_PTR(-EINVAL);
+ smmu_pmu->reg_base = devm_ioremap(dev, res_0->start,
resource_size(res_0));
+ if (!smmu_pmu->reg_base)
+ return ERR_PTR(-ENOMEM);
cfgr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_CFGR);
/* Determine if page 1 is present */
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) {
- smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);
- if (IS_ERR(smmu_pmu->reloc_base))
- return PTR_ERR(smmu_pmu->reloc_base);
+ struct resource *res_1;
+
+ res_1 = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+ if (!res_1)
+ return ERR_PTR(-EINVAL);
+ smmu_pmu->reloc_base = devm_ioremap(dev, res_1->start,
resource_size(res_1));
+ if (!smmu_pmu->reloc_base)
+ return ERR_PTR(-ENOMEM);
} else {
smmu_pmu->reloc_base = smmu_pmu->reg_base;
}
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] Revert "iommu/iova: Retry from last rb tree node if iova search fails"
This reverts commit 4e89dce725213d3d0b0475211b500eda4ef4bf2f.
We find that this patch has a great impact on performance. According to
our test: the iops decreases from 1655.6K to 893.5K, about half.
Hardware: 1 SAS expander with 12 SAS SSD
Command: Only the main parameters are listed.
fio bs=4k rw=read iodepth=128 cpus_allowed=0-127
Fixes: 4e89dce72521 ("iommu/iova: Retry from last rb tree node if iova search
fails")
Tested-by: Xiang Chen
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 23 ++-
1 file changed, 6 insertions(+), 17 deletions(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index d20b8b333d30d17..f840c7207efbced 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -185,9 +185,8 @@ static int __alloc_and_insert_iova_range(struct iova_domain
*iovad,
struct rb_node *curr, *prev;
struct iova *curr_iova;
unsigned long flags;
- unsigned long new_pfn, retry_pfn;
+ unsigned long new_pfn;
unsigned long align_mask = ~0UL;
- unsigned long high_pfn = limit_pfn, low_pfn = iovad->start_pfn;
if (size_aligned)
align_mask <<= fls_long(size - 1);
@@ -200,25 +199,15 @@ static int __alloc_and_insert_iova_range(struct
iova_domain *iovad,
curr = __get_cached_rbnode(iovad, limit_pfn);
curr_iova = rb_entry(curr, struct iova, node);
- retry_pfn = curr_iova->pfn_hi + 1;
-
-retry:
do {
- high_pfn = min(high_pfn, curr_iova->pfn_lo);
- new_pfn = (high_pfn - size) & align_mask;
+ limit_pfn = min(limit_pfn, curr_iova->pfn_lo);
+ new_pfn = (limit_pfn - size) & align_mask;
prev = curr;
curr = rb_prev(curr);
curr_iova = rb_entry(curr, struct iova, node);
- } while (curr && new_pfn <= curr_iova->pfn_hi && new_pfn >= low_pfn);
-
- if (high_pfn < size || new_pfn < low_pfn) {
- if (low_pfn == iovad->start_pfn && retry_pfn < limit_pfn) {
- high_pfn = limit_pfn;
- low_pfn = retry_pfn;
- curr = >anchor.node;
- curr_iova = rb_entry(curr, struct iova, node);
- goto retry;
- }
+ } while (curr && new_pfn <= curr_iova->pfn_hi);
+
+ if (limit_pfn < size || new_pfn < iovad->start_pfn) {
iovad->max32_alloc_size = size;
goto iova32_full;
}
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 3/3] iommu/arm-smmu-v3: Reserving the entire SMMU register space
commit 52f3fab0067d ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") only reserves the basic SMMU register space. So
the ECMDQ register space is not covered, it should be mapped again. Due
to the size of this ECMDQ resource is not fixed, depending on
SMMU_IDR6.CMDQ_CONTROL_PAGE_LOG2NUMQ. Processing its resource reservation
to avoid resource conflict with PMCG is a bit more complicated.
Therefore, the resources of the PMCG are not reserved, and the entire SMMU
resources are reserved.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
2 files changed, 4 insertions(+), 22 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index f04c55a7503c790..fde24403b06a9e3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3476,14 +3476,6 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
return err;
}
-static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t
start,
- resource_size_t size)
-{
- struct resource res = DEFINE_RES_MEM(start, size);
-
- return devm_ioremap_resource(dev, );
-}
-
static int arm_smmu_device_probe(struct platform_device *pdev)
{
int irq, ret;
@@ -3519,22 +3511,14 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
}
ioaddr = res->start;
- /*
-* Don't map the IMPLEMENTATION DEFINED regions, since they may contain
-* the PMCG registers which are reserved by the PMU driver.
-*/
- smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
+ smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
- if (arm_smmu_resource_size(smmu) > SZ_64K) {
- smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
- ARM_SMMU_REG_SZ);
- if (IS_ERR(smmu->page1))
- return PTR_ERR(smmu->page1);
- } else {
+ if (arm_smmu_resource_size(smmu) > SZ_64K)
+ smmu->page1 = smmu->base + SZ_64K;
+ else
smmu->page1 = smmu->base;
- }
/* Interrupt lines */
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index da525f46dab4fc1..63f2b476987d6ae 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -152,8 +152,6 @@
#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
-#define ARM_SMMU_REG_SZ0xe00
-
/* Common MSI config fields */
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
#define MSI_CFG2_SHGENMASK(5, 4)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 1/3] perf/smmuv3: Don't reserve the PMCG register spaces
According to the SMMUv3 specification:
Each PMCG counter group is represented by one 4KB page (Page 0) with one
optional additional 4KB page (Page 1), both of which are at IMPLEMENTATION
DEFINED base addresses.
This means that the PMCG register spaces may be within the 64KB pages of
the SMMUv3 register space. When both the SMMU and PMCG drivers reserve
their own resources, a resource conflict occurs.
To avoid this conflict, don't reserve the PMCG regions.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 27 +--
1 file changed, 25 insertions(+), 2 deletions(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 74474bb322c3f26..e5e505a0804fe53 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -761,6 +761,29 @@ static void smmu_pmu_get_acpi_options(struct smmu_pmu
*smmu_pmu)
dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options);
}
+static void __iomem *
+smmu_pmu_get_and_ioremap_resource(struct platform_device *pdev,
+ unsigned int index,
+ struct resource **res)
+{
+ void __iomem *base;
+ struct resource *r;
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, index);
+ if (!r) {
+ dev_err(>dev, "invalid resource\n");
+ return ERR_PTR(-EINVAL);
+ }
+ if (res)
+ *res = r;
+
+ base = devm_ioremap(>dev, r->start, resource_size(r));
+ if (!base)
+ return ERR_PTR(-ENOMEM);
+
+ return base;
+}
+
static int smmu_pmu_probe(struct platform_device *pdev)
{
struct smmu_pmu *smmu_pmu;
@@ -793,7 +816,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
- smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0,
_0);
+ smmu_pmu->reg_base = smmu_pmu_get_and_ioremap_resource(pdev, 0, _0);
if (IS_ERR(smmu_pmu->reg_base))
return PTR_ERR(smmu_pmu->reg_base);
@@ -801,7 +824,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
/* Determine if page 1 is present */
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) {
- smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);
+ smmu_pmu->reloc_base = smmu_pmu_get_and_ioremap_resource(pdev,
1, NULL);
if (IS_ERR(smmu_pmu->reloc_base))
return PTR_ERR(smmu_pmu->reloc_base);
} else {
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 0/3] perf/smmuv3: Don't reserve the PMCG register spaces
v2 --> v3:
Patch 3 is updated because https://lkml.org/lkml/2021/1/22/532 has been queued
in advance.
v1 --> v2:
According to Robin Murphy's suggestion: https://lkml.org/lkml/2021/1/20/470
Don't reserve the PMCG register spaces, and reserve the entire SMMU register
space.
v1:
Since the PMCG may implement its resigters space(4KB Page0 and 4KB Page1)
within the SMMUv3 64KB Page0. In this case, when the SMMUv3 driver reserves the
64KB Page0 resource in advance, the PMCG driver try to reserve its Page0 and
Page1 resources, a resource conflict occurs.
commit 52f3fab0067d6fa ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") reduce the resource reservation range of the SMMUv3
driver, it only reserves the first 0xe00 bytes in the 64KB Page0, to avoid
the above-mentioned resource conflicts.
But the SMMUv3.3 add support for ECMDQ, its registers space is also implemented
in the SMMUv3 64KB Page0. This means we need to build two separate mappings.
New features may be added in the future, and more independent mappings may be
required. The simple problem is complicated because the user expects to map the
entire SMMUv3 64KB Page0.
Therefore, the proper solution is: If the PMCG register resources are located in
the 64KB Page0 of the SMMU, the PMCG driver does not reserve the conflict
resources
when the SMMUv3 driver has reserved the conflict resources before. Instead, the
PMCG
driver only performs devm_ioremap() to ensure that it can work properly.
Zhen Lei (3):
perf/smmuv3: Don't reserve the PMCG register spaces
perf/smmuv3: Add a MODULE_SOFTDEP() to indicate dependency on SMMU
iommu/arm-smmu-v3: Reserving the entire SMMU register space
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 24
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
drivers/perf/arm_smmuv3_pmu.c | 28 ++--
3 files changed, 30 insertions(+), 24 deletions(-)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 2/3] perf/smmuv3: Add a MODULE_SOFTDEP() to indicate dependency on SMMU
The MODULE_SOFTDEP() gives user space a hint of the loading sequence. And
when command "modprobe arm_smmuv3_pmu" is executed, the arm_smmu_v3.ko is
automatically loaded in advance.
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index e5e505a0804fe53..9a305ac51208cd2 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -950,6 +950,7 @@ static void __exit arm_smmu_pmu_exit(void)
module_exit(arm_smmu_pmu_exit);
MODULE_DESCRIPTION("PMU driver for ARM SMMUv3 Performance Monitors Extension");
+MODULE_SOFTDEP("pre: arm_smmu_v3");
MODULE_AUTHOR("Neil Leeder ");
MODULE_AUTHOR("Shameer Kolothum ");
MODULE_LICENSE("GPL v2");
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 3/3] iommu/arm-smmu-v3: Reserving the entire SMMU register space
commit 52f3fab0067d ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") only reserves the basic SMMU register space. So
the ECMDQ register space is not covered, it should be mapped again. Due
to the size of this ECMDQ resource is not fixed, depending on
SMMU_IDR6.CMDQ_CONTROL_PAGE_LOG2NUMQ. Processing its resource reservation
to avoid resource conflict with PMCG is a bit more complicated.
Therefore, the resources of the PMCG are not reserved, and the entire SMMU
resources are reserved.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 28
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
2 files changed, 4 insertions(+), 26 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index bca458c00e48a8b..fde24403b06a9e3 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3476,18 +3476,6 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
return err;
}
-static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t
start,
- resource_size_t size)
-{
- struct resource res = {
- .flags = IORESOURCE_MEM,
- .start = start,
- .end = start + size - 1,
- };
-
- return devm_ioremap_resource(dev, );
-}
-
static int arm_smmu_device_probe(struct platform_device *pdev)
{
int irq, ret;
@@ -3523,22 +3511,14 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
}
ioaddr = res->start;
- /*
-* Don't map the IMPLEMENTATION DEFINED regions, since they may contain
-* the PMCG registers which are reserved by the PMU driver.
-*/
- smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
+ smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
- if (arm_smmu_resource_size(smmu) > SZ_64K) {
- smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
- ARM_SMMU_REG_SZ);
- if (IS_ERR(smmu->page1))
- return PTR_ERR(smmu->page1);
- } else {
+ if (arm_smmu_resource_size(smmu) > SZ_64K)
+ smmu->page1 = smmu->base + SZ_64K;
+ else
smmu->page1 = smmu->base;
- }
/* Interrupt lines */
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index da525f46dab4fc1..63f2b476987d6ae 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -152,8 +152,6 @@
#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
-#define ARM_SMMU_REG_SZ0xe00
-
/* Common MSI config fields */
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
#define MSI_CFG2_SHGENMASK(5, 4)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 1/3] perf/smmuv3: Don't reserve the PMCG register spaces
According to the SMMUv3 specification:
Each PMCG counter group is represented by one 4KB page (Page 0) with one
optional additional 4KB page (Page 1), both of which are at IMPLEMENTATION
DEFINED base addresses.
This means that the PMCG register spaces may be within the 64KB pages of
the SMMUv3 register space. When both the SMMU and PMCG drivers reserve
their own resources, a resource conflict occurs.
To avoid this conflict, don't reserve the PMCG regions.
Suggested-by: Robin Murphy
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 27 +--
1 file changed, 25 insertions(+), 2 deletions(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 74474bb322c3f26..e5e505a0804fe53 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -761,6 +761,29 @@ static void smmu_pmu_get_acpi_options(struct smmu_pmu
*smmu_pmu)
dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options);
}
+static void __iomem *
+smmu_pmu_get_and_ioremap_resource(struct platform_device *pdev,
+ unsigned int index,
+ struct resource **res)
+{
+ void __iomem *base;
+ struct resource *r;
+
+ r = platform_get_resource(pdev, IORESOURCE_MEM, index);
+ if (!r) {
+ dev_err(>dev, "invalid resource\n");
+ return ERR_PTR(-EINVAL);
+ }
+ if (res)
+ *res = r;
+
+ base = devm_ioremap(>dev, r->start, resource_size(r));
+ if (!base)
+ return ERR_PTR(-ENOMEM);
+
+ return base;
+}
+
static int smmu_pmu_probe(struct platform_device *pdev)
{
struct smmu_pmu *smmu_pmu;
@@ -793,7 +816,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
- smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0,
_0);
+ smmu_pmu->reg_base = smmu_pmu_get_and_ioremap_resource(pdev, 0, _0);
if (IS_ERR(smmu_pmu->reg_base))
return PTR_ERR(smmu_pmu->reg_base);
@@ -801,7 +824,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
/* Determine if page 1 is present */
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) {
- smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);
+ smmu_pmu->reloc_base = smmu_pmu_get_and_ioremap_resource(pdev,
1, NULL);
if (IS_ERR(smmu_pmu->reloc_base))
return PTR_ERR(smmu_pmu->reloc_base);
} else {
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 2/3] perf/smmuv3: Add a MODULE_SOFTDEP() to indicate dependency on SMMU
The MODULE_SOFTDEP() gives user space a hint of the loading sequence. And
when command "modprobe arm_smmuv3_pmu" is executed, the arm_smmu_v3.ko is
automatically loaded in advance.
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index e5e505a0804fe53..9a305ac51208cd2 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -950,6 +950,7 @@ static void __exit arm_smmu_pmu_exit(void)
module_exit(arm_smmu_pmu_exit);
MODULE_DESCRIPTION("PMU driver for ARM SMMUv3 Performance Monitors Extension");
+MODULE_SOFTDEP("pre: arm_smmu_v3");
MODULE_AUTHOR("Neil Leeder ");
MODULE_AUTHOR("Shameer Kolothum ");
MODULE_LICENSE("GPL v2");
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 0/3] perf/smmuv3: Don't reserve the PMCG register spaces
v1 --> v2:
According to Robin Murphy's suggestion: https://lkml.org/lkml/2021/1/20/470
Don't reserve the PMCG register spaces, and reserve the entire SMMU register
space.
v1:
Since the PMCG may implement its resigters space(4KB Page0 and 4KB Page1)
within the SMMUv3 64KB Page0. In this case, when the SMMUv3 driver reserves the
64KB Page0 resource in advance, the PMCG driver try to reserve its Page0 and
Page1 resources, a resource conflict occurs.
commit 52f3fab0067d6fa ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") reduce the resource reservation range of the SMMUv3
driver, it only reserves the first 0xe00 bytes in the 64KB Page0, to avoid
the above-mentioned resource conflicts.
But the SMMUv3.3 add support for ECMDQ, its registers space is also implemented
in the SMMUv3 64KB Page0. This means we need to build two separate mappings.
New features may be added in the future, and more independent mappings may be
required. The simple problem is complicated because the user expects to map the
entire SMMUv3 64KB Page0.
Therefore, the proper solution is: If the PMCG register resources are located in
the 64KB Page0 of the SMMU, the PMCG driver does not reserve the conflict
resources
when the SMMUv3 driver has reserved the conflict resources before. Instead, the
PMCG
driver only performs devm_ioremap() to ensure that it can work properly.
Zhen Lei (3):
perf/smmuv3: Don't reserve the PMCG register spaces
perf/smmuv3: Add a MODULE_SOFTDEP() to indicate dependency on SMMU
iommu/arm-smmu-v3: Reserving the entire SMMU register space
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 28
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 --
drivers/perf/arm_smmuv3_pmu.c | 28 ++--
3 files changed, 30 insertions(+), 28 deletions(-)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu-v3: Use DEFINE_RES_MEM() to simplify code
No functional change.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 6 +-
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index bca458c00e48a8b..f04c55a7503c790 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3479,11 +3479,7 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t
start,
resource_size_t size)
{
- struct resource res = {
- .flags = IORESOURCE_MEM,
- .start = start,
- .end = start + size - 1,
- };
+ struct resource res = DEFINE_RES_MEM(start, size);
return devm_ioremap_resource(dev, );
}
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 0/2] Use another method to avoid resource conflicts between the SMMU and PMCG drivers
Since the PMCG may implement its resigters space(4KB Page0 and 4KB Page1)
within the SMMUv3 64KB Page0. In this case, when the SMMUv3 driver reserves the
64KB Page0 resource in advance, the PMCG driver try to reserve its Page0 and
Page1 resources, a resource conflict occurs.
commit 52f3fab0067d6fa ("iommu/arm-smmu-v3: Don't reserve implementation
defined register space") reduce the resource reservation range of the SMMUv3
driver, it only reserves the first 0xe00 bytes in the 64KB Page0, to avoid
the above-mentioned resource conflicts.
But the SMMUv3.3 add support for ECMDQ, its registers space is also implemented
in the SMMUv3 64KB Page0. This means we need to build two separate mappings.
New features may be added in the future, and more independent mappings may be
required. The simple problem is complicated because the user expects to map the
entire SMMUv3 64KB Page0.
Therefore, the proper solution is: If the PMCG register resources are located in
the 64KB Page0 of the SMMU, the PMCG driver does not reserve the conflict
resources
when the SMMUv3 driver has reserved the conflict resources before. Instead, the
PMCG
driver only performs devm_ioremap() to ensure that it can work properly.
Zhen Lei (2):
perf/smmuv3: Don't reserve the register space that overlaps with the
SMMUv3
Revert "iommu/arm-smmu-v3: Don't reserve implementation defined
register space"
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 32 +++---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 ---
drivers/perf/arm_smmuv3_pmu.c | 42 +++--
3 files changed, 44 insertions(+), 33 deletions(-)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 2/2] Revert "iommu/arm-smmu-v3: Don't reserve implementation defined register space"
This reverts commit 52f3fab0067d6fa9e99c1b7f63265dd48ca76046.
This problem has been fixed by another patch. The original method had side
effects, it was not mapped to the user-specified resource size. The code
will become more complex when ECMDQ is supported later.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 32 -
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 3 ---
2 files changed, 4 insertions(+), 31 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 8ca7415d785d9bf..477f473842e5272 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -91,8 +91,9 @@ struct arm_smmu_option_prop {
static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
struct arm_smmu_device *smmu)
{
- if (offset > SZ_64K)
- return smmu->page1 + offset - SZ_64K;
+ if ((offset > SZ_64K) &&
+ (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
+ offset -= SZ_64K;
return smmu->base + offset;
}
@@ -3486,18 +3487,6 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
return err;
}
-static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t
start,
- resource_size_t size)
-{
- struct resource res = {
- .flags = IORESOURCE_MEM,
- .start = start,
- .end = start + size - 1,
- };
-
- return devm_ioremap_resource(dev, );
-}
-
static int arm_smmu_device_probe(struct platform_device *pdev)
{
int irq, ret;
@@ -3533,23 +3522,10 @@ static int arm_smmu_device_probe(struct platform_device
*pdev)
}
ioaddr = res->start;
- /*
-* Don't map the IMPLEMENTATION DEFINED regions, since they may contain
-* the PMCG registers which are reserved by the PMU driver.
-*/
- smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
+ smmu->base = devm_ioremap_resource(dev, res);
if (IS_ERR(smmu->base))
return PTR_ERR(smmu->base);
- if (arm_smmu_resource_size(smmu) > SZ_64K) {
- smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
- ARM_SMMU_REG_SZ);
- if (IS_ERR(smmu->page1))
- return PTR_ERR(smmu->page1);
- } else {
- smmu->page1 = smmu->base;
- }
-
/* Interrupt lines */
irq = platform_get_irq_byname_optional(pdev, "combined");
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 96c2e9565e00282..0c3090c60840c22 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -152,8 +152,6 @@
#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
-#define ARM_SMMU_REG_SZ0xe00
-
/* Common MSI config fields */
#define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
#define MSI_CFG2_SHGENMASK(5, 4)
@@ -584,7 +582,6 @@ struct arm_smmu_strtab_cfg {
struct arm_smmu_device {
struct device *dev;
void __iomem*base;
- void __iomem*page1;
#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/2] perf/smmuv3: Don't reserve the register space that overlaps with the SMMUv3
Some SMMUv3 implementation embed the Perf Monitor Group Registers (PMCG)
inside the first 64kB region of the SMMU. Since SMMU and PMCG are managed
by two separate drivers, and this driver depends on ARM_SMMU_V3, so the
SMMU driver reserves the corresponding resource first, this driver should
not reserve the corresponding resource again. Otherwise, a resource
reservation conflict is reported during boot.
Signed-off-by: Zhen Lei
---
drivers/perf/arm_smmuv3_pmu.c | 42 --
1 file changed, 40 insertions(+), 2 deletions(-)
diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c
index 74474bb322c3f26..dcce085431c6ce8 100644
--- a/drivers/perf/arm_smmuv3_pmu.c
+++ b/drivers/perf/arm_smmuv3_pmu.c
@@ -761,6 +761,44 @@ static void smmu_pmu_get_acpi_options(struct smmu_pmu
*smmu_pmu)
dev_notice(smmu_pmu->dev, "option mask 0x%x\n", smmu_pmu->options);
}
+static void __iomem *
+smmu_pmu_get_and_ioremap_resource(struct platform_device *pdev,
+ unsigned int index,
+ struct resource **out_res)
+{
+ int ret;
+ void __iomem *base;
+ struct resource *res;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, index);
+ if (!res) {
+ dev_err(>dev, "invalid resource\n");
+ return IOMEM_ERR_PTR(-EINVAL);
+ }
+ if (out_res)
+ *out_res = res;
+
+ ret = region_intersects(res->start, resource_size(res),
+ IORESOURCE_MEM, IORES_DESC_NONE);
+ if (ret == REGION_INTERSECTS) {
+ /*
+* The resource has already been reserved by the SMMUv3 driver.
+* Don't reserve it again, just do devm_ioremap().
+*/
+ base = devm_ioremap(>dev, res->start, resource_size(res));
+ } else {
+ /*
+* The resource may have not been reserved by any driver, or
+* has been reserved but not type IORESOURCE_MEM. In the latter
+* case, devm_ioremap_resource() reports a conflict and returns
+* IOMEM_ERR_PTR(-EBUSY).
+*/
+ base = devm_ioremap_resource(>dev, res);
+ }
+
+ return base;
+}
+
static int smmu_pmu_probe(struct platform_device *pdev)
{
struct smmu_pmu *smmu_pmu;
@@ -793,7 +831,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
};
- smmu_pmu->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0,
_0);
+ smmu_pmu->reg_base = smmu_pmu_get_and_ioremap_resource(pdev, 0, _0);
if (IS_ERR(smmu_pmu->reg_base))
return PTR_ERR(smmu_pmu->reg_base);
@@ -801,7 +839,7 @@ static int smmu_pmu_probe(struct platform_device *pdev)
/* Determine if page 1 is present */
if (cfgr & SMMU_PMCG_CFGR_RELOC_CTRS) {
- smmu_pmu->reloc_base = devm_platform_ioremap_resource(pdev, 1);
+ smmu_pmu->reloc_base = smmu_pmu_get_and_ioremap_resource(pdev,
1, NULL);
if (IS_ERR(smmu_pmu->reloc_base))
return PTR_ERR(smmu_pmu->reloc_base);
} else {
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH 1/1] iommu/arm-smmu-v3: add support for BBML
When changing from a set of pages/smaller blocks to a larger block for an
address, the software should follow the sequence of BBML processing.
When changing from a block to a set of pages/smaller blocks for an
address, there's no need to use nT bit. If an address in the large block
is accessed before page table switching, the TLB caches the large block
mapping. After the page table is switched and before TLB invalidation
finished, new access requests are still based on large block mapping.
After the block or page is invalidated, the system reads the small block
or page mapping from the memory; If the address in the large block is not
accessed before page table switching, the TLB has no cache. After the
page table is switched, a new access is initiated to read the small block
or page mapping from the memory.
Signed-off-by: Zhen Lei
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 2 +
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 2 +
drivers/iommu/io-pgtable-arm.c | 46 -
include/linux/io-pgtable.h | 1 +
4 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index e634bbe60573..14a1a11565fb 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -1977,6 +1977,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain
*domain,
.coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
.tlb= _smmu_flush_ops,
.iommu_dev = smmu->dev,
+ .bbml = smmu->bbml,
};
if (smmu_domain->non_strict)
@@ -3291,6 +3292,7 @@ static int arm_smmu_device_hw_probe(struct
arm_smmu_device *smmu)
/* IDR3 */
reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
+ smmu->bbml = FIELD_GET(IDR3_BBML, reg);
if (FIELD_GET(IDR3_RIL, reg))
smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index d4b7f40ccb02..aa7eb460fa09 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -51,6 +51,7 @@
#define IDR1_SIDSIZE GENMASK(5, 0)
#define ARM_SMMU_IDR3 0xc
+#define IDR3_BBML GENMASK(12, 11)
#define IDR3_RIL (1 << 10)
#define ARM_SMMU_IDR5 0x14
@@ -617,6 +618,7 @@ struct arm_smmu_device {
int gerr_irq;
int combined_irq;
+ int bbml;
unsigned long ias; /* IPA */
unsigned long oas; /* PA */
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index a7a9bc08dcd1..341581337ad0 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -72,6 +72,7 @@
#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
#define ARM_LPAE_PTE_XN(((arm_lpae_iopte)3) << 53)
+#define ARM_LPAE_PTE_nT(((arm_lpae_iopte)1) << 16)
#define ARM_LPAE_PTE_AF(((arm_lpae_iopte)1) << 10)
#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
@@ -255,7 +256,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable
*data,
static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
phys_addr_t paddr, arm_lpae_iopte prot,
- int lvl, arm_lpae_iopte *ptep)
+ int lvl, arm_lpae_iopte *ptep, arm_lpae_iopte
nT)
{
arm_lpae_iopte pte = prot;
@@ -265,37 +266,60 @@ static void __arm_lpae_init_pte(struct
arm_lpae_io_pgtable *data,
pte |= ARM_LPAE_PTE_TYPE_BLOCK;
pte |= paddr_to_iopte(paddr, data);
+ pte |= nT;
__arm_lpae_set_pte(ptep, pte, >iop.cfg);
}
+static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
+ arm_lpae_iopte *ptep);
static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
unsigned long iova, phys_addr_t paddr,
arm_lpae_iopte prot, int lvl,
arm_lpae_iopte *ptep)
{
arm_lpae_iopte pte = *ptep;
+ struct io_pgtable_cfg *cfg = >iop.cfg;
if (iopte_leaf(pte, lvl, data->iop.fmt)) {
/* We require an unmap first */
WARN_ON(!selftest_running);
return -EEXIST;
} else if (iopte_type(pte, lvl) == ARM_LPAE_PTE_TYPE_TABLE) {
-
[PATCH v3 0/2] improve the concurrency of arm_smmu_atc_inv_domain()
v2 --> v3: As Will Deacon's suggestion, I changed the lock type of arm_smmu_domain.devices_lock from spinlock_t to rwlock_t, and I saw that the performance is all right. And further use nr_ats_masters to quickly check have no obvious effect, so I drop it. Here is the performance data tested on my board: Withou any change: Jobs: 24 (f=24): [0.1% done] [9798M/0K /s] [2392K/0 iops] [09h:59m:13s] Jobs: 24 (f=24): [0.1% done] [9782M/0K /s] [2388K/0 iops] [09h:59m:12s] Jobs: 24 (f=24): [0.2% done] [9825M/0K /s] [2399K/0 iops] [09h:59m:11s] Jobs: 24 (f=24): [0.2% done] [9836M/0K /s] [2401K/0 iops] [09h:59m:10s] Change lock type from spinlock_t to rwlock_t: Jobs: 24 (f=24): [0.1% done] [10996M/0K /s] [2685K/0 iops] [09h:59m:13s] Jobs: 24 (f=24): [0.1% done] [10817M/0K /s] [2641K/0 iops] [09h:59m:12s] Jobs: 24 (f=24): [0.2% done] [11083M/0K /s] [2706K/0 iops] [09h:59m:11s] Jobs: 24 (f=24): [0.2% done] [10603M/0K /s] [2589K/0 iops] [09h:59m:10s] Use nr_ats_masters: Jobs: 24 (f=24): [0.2% done] [11105M/0K /s] [2711K/0 iops] [eta 09h:58m:40s] Jobs: 24 (f=24): [0.2% done] [10511M/0K /s] [2566K/0 iops] [eta 09h:58m:39s] Jobs: 24 (f=24): [0.2% done] [10560M/0K /s] [2578K/0 iops] [eta 09h:58m:38s] Jobs: 24 (f=24): [0.2% done] [10494M/0K /s] [2562K/0 iops] [eta 09h:58m:37s] Jobs: 24 (f=24): [0.2% done] [10528M/0K /s] [2570K/0 iops] [eta 09h:58m:36s] Jobs: 24 (f=24): [0.3% done] [10638M/0K /s] [2597K/0 iops] [eta 09h:58m:35s] Jobs: 24 (f=24): [0.3% done] [11158M/0K /s] [2724K/0 iops] [eta 09h:58m:34s] Jobs: 24 (f=24): [0.3% done] [11386M/0K /s] [2780K/0 iops] [eta 09h:58m:32s] Jobs: 24 (f=24): [0.3% done] [8M/0K /s] [2714K/0 iops] [eta 09h:58m:32s] Jobs: 24 (f=24): [0.3% done] [11031M/0K /s] [2693K/0 iops] [eta 09h:58m:31s] Jobs: 24 (f=24): [0.3% done] [11361M/0K /s] [2774K/0 iops] [eta 09h:58m:30s] v1 --> v2: 1. change the type of nr_ats_masters from atomic_t to int, and move its ind/dec operation from arm_smmu_enable_ats()/arm_smmu_disable_ats() to arm_smmu_attach_dev()/arm_smmu_detach_dev(), and protected by "spin_lock_irqsave(_domain->devices_lock, flags);" Zhen Lei (2): iommu/arm-smmu-v3: don't add a master into smmu_domain before it's ready iommu/arm-smmu-v3: change the lock type of arm_smmu_domain.devices_lock drivers/iommu/arm-smmu-v3.c | 20 ++-- 1 file changed, 10 insertions(+), 10 deletions(-) -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 1/2] iommu/arm-smmu-v3: don't add a master into smmu_domain before it's ready
Once a master has been added into smmu_domain->devices, it may immediately be scaned in arm_smmu_unmap()-->arm_smmu_atc_inv_domain(). From a logical point of view, the master should be added into smmu_domain after it has completely initialized. Signed-off-by: Zhen Lei --- drivers/iommu/arm-smmu-v3.c | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index c5c93e48b4dbdf7..e0dcc5d27291f8b 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1958,10 +1958,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) master->domain = smmu_domain; - spin_lock_irqsave(_domain->devices_lock, flags); - list_add(>domain_head, _domain->devices); - spin_unlock_irqrestore(_domain->devices_lock, flags); - if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) arm_smmu_enable_ats(master); @@ -1969,6 +1965,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_write_ctx_desc(smmu, _domain->s1_cfg); arm_smmu_install_ste_for_dev(master); + + spin_lock_irqsave(_domain->devices_lock, flags); + list_add(>domain_head, _domain->devices); + spin_unlock_irqrestore(_domain->devices_lock, flags); out_unlock: mutex_unlock(_domain->init_mutex); return ret; -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v3 2/2] iommu/arm-smmu-v3: change the lock type of arm_smmu_domain.devices_lock
A master add into or remove from smmu_domain->devices only happened in
arm_smmu_attach_dev()/arm_smmu_detach_dev(), the frequency of these
operations is very low. But we traverse smmu_domain->devices list in
arm_smmu_atc_inv_domain() are frequent. So change the protection from
spinlock to rwlock can improve concurrency, especially for the smmu
domain without ATS masters.
By the way, the cmdq has its own lock, so this change is safe.
Here is the performance data tested on my board:
Before:
Jobs: 24 (f=24): [0.1% done] [9798M/0K /s] [2392K/0 iops] [09h:59m:13s]
Jobs: 24 (f=24): [0.1% done] [9782M/0K /s] [2388K/0 iops] [09h:59m:12s]
Jobs: 24 (f=24): [0.2% done] [9825M/0K /s] [2399K/0 iops] [09h:59m:11s]
Jobs: 24 (f=24): [0.2% done] [9836M/0K /s] [2401K/0 iops] [09h:59m:10s]
After:
Jobs: 24 (f=24): [0.1% done] [10996M/0K /s] [2685K/0 iops] [09h:59m:13s]
Jobs: 24 (f=24): [0.1% done] [10817M/0K /s] [2641K/0 iops] [09h:59m:12s]
Jobs: 24 (f=24): [0.2% done] [11083M/0K /s] [2706K/0 iops] [09h:59m:11s]
Jobs: 24 (f=24): [0.2% done] [10603M/0K /s] [2589K/0 iops] [09h:59m:10s]
Signed-off-by: Zhen Lei
Suggested-by: Will Deacon
---
drivers/iommu/arm-smmu-v3.c | 16
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index e0dcc5d27291f8b..eded2e7a5a0c444 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -641,7 +641,7 @@ struct arm_smmu_domain {
struct iommu_domain domain;
struct list_headdevices;
- spinlock_t devices_lock;
+ rwlock_tdevices_lock;
};
struct arm_smmu_option_prop {
@@ -1536,10 +1536,10 @@ static int arm_smmu_atc_inv_domain(struct
arm_smmu_domain *smmu_domain,
arm_smmu_atc_inv_to_cmd(ssid, iova, size, );
- spin_lock_irqsave(_domain->devices_lock, flags);
+ read_lock_irqsave(_domain->devices_lock, flags);
list_for_each_entry(master, _domain->devices, domain_head)
ret |= arm_smmu_atc_inv_master(master, );
- spin_unlock_irqrestore(_domain->devices_lock, flags);
+ read_unlock_irqrestore(_domain->devices_lock, flags);
return ret ? -ETIMEDOUT : 0;
}
@@ -1648,7 +1648,7 @@ static struct iommu_domain
*arm_smmu_domain_alloc(unsigned type)
mutex_init(_domain->init_mutex);
INIT_LIST_HEAD(_domain->devices);
- spin_lock_init(_domain->devices_lock);
+ rwlock_init(_domain->devices_lock);
return _domain->domain;
}
@@ -1911,9 +1911,9 @@ static void arm_smmu_detach_dev(struct arm_smmu_master
*master)
if (!smmu_domain)
return;
- spin_lock_irqsave(_domain->devices_lock, flags);
+ write_lock_irqsave(_domain->devices_lock, flags);
list_del(>domain_head);
- spin_unlock_irqrestore(_domain->devices_lock, flags);
+ write_unlock_irqrestore(_domain->devices_lock, flags);
master->domain = NULL;
arm_smmu_install_ste_for_dev(master);
@@ -1966,9 +1966,9 @@ static int arm_smmu_attach_dev(struct iommu_domain
*domain, struct device *dev)
arm_smmu_install_ste_for_dev(master);
- spin_lock_irqsave(_domain->devices_lock, flags);
+ write_lock_irqsave(_domain->devices_lock, flags);
list_add(>domain_head, _domain->devices);
- spin_unlock_irqrestore(_domain->devices_lock, flags);
+ write_unlock_irqrestore(_domain->devices_lock, flags);
out_unlock:
mutex_unlock(_domain->init_mutex);
return ret;
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 0/2] iommu/iova: enhance the rcache optimization
v1 --> v2
1. I did not chagne the patches but added this cover-letter.
2. Add a batch of reviewers base on
9257b4a206fc ("iommu/iova: introduce per-cpu caching to iova allocation")
3. I described the problem I met in patch 2, but I hope below brief description
can help people to quickly understand.
Suppose there are six rcache sizes, each size can maximum hold 1 IOVAs.
| 4K | 8K | 16K | 32K | 64K | 128K |
| 1 | 9000 | 8500 | 8600 | 9200 | 7000 |
As the above map displayed, the whole rcache buffered too many IOVAs. Now,
the
worst case can be coming, suppose we need 2 4K IOVAs at one time. That
means
1 IOVAs can be allocated from rcache, but another 1 IOVAs should be
allocated from RB tree base on alloc_iova() function. But the RB tree
currently
have at least (9000 + 8500 + 8600 + 9200 + 7000) = 42300 nodes. The average
speed
of RB tree traverse will be very slow. For my test scenario, the 4K size
IOVAs are
frequently used, but others are not. So similarly, when the 2 4K IOVAs
are
continuous freed, the first 1 IOVAs can be quickly buffered, but the
other
1 IOVAs can not.
Zhen Lei (2):
iommu/iova: introduce iova_magazine_compact_pfns()
iommu/iova: enhance the rcache optimization
drivers/iommu/iova.c | 100 +++
include/linux/iova.h | 1 +
2 files changed, 95 insertions(+), 6 deletions(-)
--
1.8.3
[PATCH v2 1/2] iommu/iova: introduce iova_magazine_compact_pfns()
iova_magazine_free_pfns() can only free the whole magazine buffer, add
iova_magazine_compact_pfns() to support free part of it.
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 17 -
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 3e1a8a6755723a9..4b7a9efa0ef40af 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -795,18 +795,19 @@ static void iova_magazine_free(struct iova_magazine *mag)
kfree(mag);
}
-static void
-iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
+static void iova_magazine_compact_pfns(struct iova_magazine *mag,
+ struct iova_domain *iovad,
+ unsigned long newsize)
{
unsigned long flags;
int i;
- if (!mag)
+ if (!mag || mag->size <= newsize)
return;
spin_lock_irqsave(>iova_rbtree_lock, flags);
- for (i = 0 ; i < mag->size; ++i) {
+ for (i = newsize; i < mag->size; ++i) {
struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
BUG_ON(!iova);
@@ -815,7 +816,13 @@ static void iova_magazine_free(struct iova_magazine *mag)
spin_unlock_irqrestore(>iova_rbtree_lock, flags);
- mag->size = 0;
+ mag->size = newsize;
+}
+
+static void
+iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
+{
+ iova_magazine_compact_pfns(mag, iovad, 0);
}
static bool iova_magazine_full(struct iova_magazine *mag)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 2/2] iommu/iova: enhance the rcache optimization
The rcache method caches the freed IOVAs, to improve the performance of
IOVAs allocation and release. This is usually okay, but it maybe declined
in some special scenarios.
For example, currently the IOVA_RANGE_CACHE_MAX_SIZE is 6, and for ecch
size, contains: MAX_GLOBAL_MAGS=32 shareable depot magazines, each vcpu
has two magazines(cpu_rcaches->loaded and cpu_rcaches->prev). In an
extreme case, it can max cache ((num_possible_cpus() * 2 + 32) * 128 * 6)
IOVAs, it's very large. The worst case happens when the depot magazines
of a certain size(usually 4K) is full, further free_iova_fast() invoking
will cause iova_magazine_free_pfns() to be called. As the above saied,
too many IOVAs buffered, so that the RB tree is very large, the
iova_magazine_free_pfns()-->private_find_iova(), and the missed iova
allocation: alloc_iova()-->__alloc_and_insert_iova_range() will spend too
much time. And that the current rcache method have no cleanup operation,
the buffered IOVAs will only increase but not decrease.
For my FIO stress test scenario, the performance drop about 35%, and can
not recover even if re-execute the test cases.
Jobs: 21 (f=21): [2.3% done] [8887M/0K /s] [2170K/0 iops]
Jobs: 21 (f=21): [2.3% done] [8902M/0K /s] [2173K/0 iops]
Jobs: 21 (f=21): [2.3% done] [6010M/0K /s] [1467K/0 iops]
Jobs: 21 (f=21): [2.3% done] [5397M/0K /s] [1318K/0 iops]
So that, I add the statistic about the rcache, when the above case
happened, release the IOVAs which are not hit.
Jobs: 21 (f=21): [100.0% done] [10324M/0K /s] [2520K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10290M/0K /s] [2512K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10035M/0K /s] [2450K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10214M/0K /s] [2494K/0 iops]
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 83 +++-
include/linux/iova.h | 1 +
2 files changed, 83 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 4b7a9efa0ef40af..f3828f4add25375 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -23,6 +23,8 @@ static unsigned long iova_rcache_get(struct iova_domain
*iovad,
unsigned long limit_pfn);
static void init_iova_rcaches(struct iova_domain *iovad);
static void free_iova_rcaches(struct iova_domain *iovad);
+static void iova_compact_rcache(struct iova_domain *iovad,
+ struct iova_rcache *curr_rcache);
static void fq_destroy_all_entries(struct iova_domain *iovad);
static void fq_flush_timeout(struct timer_list *t);
@@ -781,6 +783,8 @@ struct iova_magazine {
struct iova_cpu_rcache {
spinlock_t lock;
+ bool prev_mag_hit;
+ unsigned long nr_hit;
struct iova_magazine *loaded;
struct iova_magazine *prev;
};
@@ -934,6 +938,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
if (mag_to_free) {
iova_magazine_free_pfns(mag_to_free, iovad);
iova_magazine_free(mag_to_free);
+ iova_compact_rcache(iovad, rcache);
}
return can_insert;
@@ -971,18 +976,22 @@ static unsigned long __iova_rcache_get(struct iova_rcache
*rcache,
} else if (!iova_magazine_empty(cpu_rcache->prev)) {
swap(cpu_rcache->prev, cpu_rcache->loaded);
has_pfn = true;
+ cpu_rcache->prev_mag_hit = true;
} else {
spin_lock(>lock);
if (rcache->depot_size > 0) {
iova_magazine_free(cpu_rcache->loaded);
cpu_rcache->loaded =
rcache->depot[--rcache->depot_size];
has_pfn = true;
+ rcache->depot_mags_hit = true;
}
spin_unlock(>lock);
}
- if (has_pfn)
+ if (has_pfn) {
+ cpu_rcache->nr_hit++;
iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
+ }
spin_unlock_irqrestore(_rcache->lock, flags);
@@ -1049,5 +1058,77 @@ void free_cpu_cached_iovas(unsigned int cpu, struct
iova_domain *iovad)
}
}
+static void iova_compact_percpu_mags(struct iova_domain *iovad,
+struct iova_rcache *rcache)
+{
+ unsigned int cpu;
+
+ for_each_possible_cpu(cpu) {
+ unsigned long flags;
+ struct iova_cpu_rcache *cpu_rcache;
+
+ cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
+
+ spin_lock_irqsave(_rcache->lock, flags);
+ if (!cpu_rcache->prev_mag_hit)
+ iova_magazine_free_pfns(cpu_rcache->prev, iovad);
+
+ if (cpu_rcache->nr_hit < IOVA_MAG_SIZE)
+ iova_magazine_compact_pfns(cpu_rcache->loaded,
+ iovad,
+
[PATCH v2 0/2] add nr_ats_masters for quickly check
v1 --> v2: 1. change the type of nr_ats_masters from atomic_t to int, and move its ind/dec operation from arm_smmu_enable_ats()/arm_smmu_disable_ats() to arm_smmu_attach_dev()/arm_smmu_detach_dev(), and protected by "spin_lock_irqsave(_domain->devices_lock, flags);" Zhen Lei (2): iommu/arm-smmu-v3: don't add a master into smmu_domain before it's ready iommu/arm-smmu-v3: add nr_ats_masters for quickly check drivers/iommu/arm-smmu-v3.c | 22 +- 1 file changed, 17 insertions(+), 5 deletions(-) -- 1.8.3
[PATCH v2 2/2] iommu/arm-smmu-v3: add nr_ats_masters for quickly check
When (smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS) is true, even if a
smmu domain does not contain any ats master, the operations of
arm_smmu_atc_inv_to_cmd() and lock protection in arm_smmu_atc_inv_domain()
are always executed. This will impact performance, especially in
multi-core and stress scenarios. For my FIO test scenario, about 8%
performance reduced.
In fact, we can use a struct member to record how many ats masters that
the smmu contains. And check that without traverse the list and check all
masters one by one in the lock protection.
Fixes: 9ce27afc0830 ("iommu/arm-smmu-v3: Add support for PCI ATS")
Signed-off-by: Zhen Lei
---
drivers/iommu/arm-smmu-v3.c | 14 +-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 29056d9bb12aa01..154334d3310c9b8 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -631,6 +631,7 @@ struct arm_smmu_domain {
struct io_pgtable_ops *pgtbl_ops;
boolnon_strict;
+ int nr_ats_masters;
enum arm_smmu_domain_stage stage;
union {
@@ -1531,7 +1532,16 @@ static int arm_smmu_atc_inv_domain(struct
arm_smmu_domain *smmu_domain,
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_master *master;
- if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
+ /*
+* The protectiom of spinlock(_domain->devices_lock) is omitted.
+* Because for a given master, its map/unmap operations should only be
+* happened after it has been attached and before it has been detached.
+* So that, if at least one master need to be atc invalidated, the
+* value of smmu_domain->nr_ats_masters can not be zero.
+*
+* This can alleviate performance loss in multi-core scenarios.
+*/
+ if (!smmu_domain->nr_ats_masters)
return 0;
arm_smmu_atc_inv_to_cmd(ssid, iova, size, );
@@ -1913,6 +1923,7 @@ static void arm_smmu_detach_dev(struct arm_smmu_master
*master)
spin_lock_irqsave(_domain->devices_lock, flags);
list_del(>domain_head);
+ smmu_domain->nr_ats_masters--;
spin_unlock_irqrestore(_domain->devices_lock, flags);
master->domain = NULL;
@@ -1968,6 +1979,7 @@ static int arm_smmu_attach_dev(struct iommu_domain
*domain, struct device *dev)
spin_lock_irqsave(_domain->devices_lock, flags);
list_add(>domain_head, _domain->devices);
+ smmu_domain->nr_ats_masters++;
spin_unlock_irqrestore(_domain->devices_lock, flags);
out_unlock:
mutex_unlock(_domain->init_mutex);
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v2 1/2] iommu/arm-smmu-v3: don't add a master into smmu_domain before it's ready
Once a master has been added into smmu_domain->devices, it may immediately be scaned in arm_smmu_unmap()-->arm_smmu_atc_inv_domain(). From a logical point of view, the master should be added into smmu_domain after it has been completely initialized. Signed-off-by: Zhen Lei --- drivers/iommu/arm-smmu-v3.c | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index a9a9fabd396804a..29056d9bb12aa01 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1958,10 +1958,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) master->domain = smmu_domain; - spin_lock_irqsave(_domain->devices_lock, flags); - list_add(>domain_head, _domain->devices); - spin_unlock_irqrestore(_domain->devices_lock, flags); - if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) arm_smmu_enable_ats(master); @@ -1969,6 +1965,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_write_ctx_desc(smmu, _domain->s1_cfg); arm_smmu_install_ste_for_dev(master); + + spin_lock_irqsave(_domain->devices_lock, flags); + list_add(>domain_head, _domain->devices); + spin_unlock_irqrestore(_domain->devices_lock, flags); out_unlock: mutex_unlock(_domain->init_mutex); return ret; -- 1.8.3
[PATCH 2/2] iommu/iova: enhance the rcache optimization
The rcache method caches the freed IOVAs, to improve the performance of
IOVAs allocation and release. This is usually okay, but it maybe declined
in some special scenarios.
For example, currently the IOVA_RANGE_CACHE_MAX_SIZE is 6, and for ecch
size, contains: MAX_GLOBAL_MAGS=32 shareable depot magazines, each vcpu
has two magazines(cpu_rcaches->loaded and cpu_rcaches->prev). In an
extreme case, it can max cache ((num_possible_cpus() * 2 + 32) * 128 * 6)
IOVAs, it's very large. The worst case happens when the depot magazines
of a certain size(usually 4K) is full, further free_iova_fast() invoking
will cause iova_magazine_free_pfns() to be called. As the above saied,
too many IOVAs buffered, so that the RB tree is very large, the
iova_magazine_free_pfns()-->private_find_iova(), and the missed iova
allocation: alloc_iova()-->__alloc_and_insert_iova_range() will spend too
much time. And that the current rcache method have no cleanup operation,
the buffered IOVAs will only increase but not decrease.
For my FIO stress test scenario, the performance drop about 35%, and can
not recover even if re-execute the test cases.
Jobs: 21 (f=21): [2.3% done] [8887M/0K /s] [2170K/0 iops]
Jobs: 21 (f=21): [2.3% done] [8902M/0K /s] [2173K/0 iops]
Jobs: 21 (f=21): [2.3% done] [6010M/0K /s] [1467K/0 iops]
Jobs: 21 (f=21): [2.3% done] [5397M/0K /s] [1318K/0 iops]
So that, I add the statistic about the rcache, when the above case
happened, release the IOVAs which are not hit.
Jobs: 21 (f=21): [100.0% done] [10324M/0K /s] [2520K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10290M/0K /s] [2512K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10035M/0K /s] [2450K/0 iops]
Jobs: 21 (f=21): [100.0% done] [10214M/0K /s] [2494K/0 iops]
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 83 +++-
include/linux/iova.h | 1 +
2 files changed, 83 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 4b7a9efa0ef40af..f3828f4add25375 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -23,6 +23,8 @@ static unsigned long iova_rcache_get(struct iova_domain
*iovad,
unsigned long limit_pfn);
static void init_iova_rcaches(struct iova_domain *iovad);
static void free_iova_rcaches(struct iova_domain *iovad);
+static void iova_compact_rcache(struct iova_domain *iovad,
+ struct iova_rcache *curr_rcache);
static void fq_destroy_all_entries(struct iova_domain *iovad);
static void fq_flush_timeout(struct timer_list *t);
@@ -781,6 +783,8 @@ struct iova_magazine {
struct iova_cpu_rcache {
spinlock_t lock;
+ bool prev_mag_hit;
+ unsigned long nr_hit;
struct iova_magazine *loaded;
struct iova_magazine *prev;
};
@@ -934,6 +938,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
if (mag_to_free) {
iova_magazine_free_pfns(mag_to_free, iovad);
iova_magazine_free(mag_to_free);
+ iova_compact_rcache(iovad, rcache);
}
return can_insert;
@@ -971,18 +976,22 @@ static unsigned long __iova_rcache_get(struct iova_rcache
*rcache,
} else if (!iova_magazine_empty(cpu_rcache->prev)) {
swap(cpu_rcache->prev, cpu_rcache->loaded);
has_pfn = true;
+ cpu_rcache->prev_mag_hit = true;
} else {
spin_lock(>lock);
if (rcache->depot_size > 0) {
iova_magazine_free(cpu_rcache->loaded);
cpu_rcache->loaded =
rcache->depot[--rcache->depot_size];
has_pfn = true;
+ rcache->depot_mags_hit = true;
}
spin_unlock(>lock);
}
- if (has_pfn)
+ if (has_pfn) {
+ cpu_rcache->nr_hit++;
iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn);
+ }
spin_unlock_irqrestore(_rcache->lock, flags);
@@ -1049,5 +1058,77 @@ void free_cpu_cached_iovas(unsigned int cpu, struct
iova_domain *iovad)
}
}
+static void iova_compact_percpu_mags(struct iova_domain *iovad,
+struct iova_rcache *rcache)
+{
+ unsigned int cpu;
+
+ for_each_possible_cpu(cpu) {
+ unsigned long flags;
+ struct iova_cpu_rcache *cpu_rcache;
+
+ cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
+
+ spin_lock_irqsave(_rcache->lock, flags);
+ if (!cpu_rcache->prev_mag_hit)
+ iova_magazine_free_pfns(cpu_rcache->prev, iovad);
+
+ if (cpu_rcache->nr_hit < IOVA_MAG_SIZE)
+ iova_magazine_compact_pfns(cpu_rcache->loaded,
+ iovad,
+
[PATCH 1/2] iommu/iova: introduce iova_magazine_compact_pfns()
iova_magazine_free_pfns() can only free the whole magazine buffer, add
iova_magazine_compact_pfns() to support free part of it.
Signed-off-by: Zhen Lei
---
drivers/iommu/iova.c | 17 -
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
index 3e1a8a6755723a9..4b7a9efa0ef40af 100644
--- a/drivers/iommu/iova.c
+++ b/drivers/iommu/iova.c
@@ -795,18 +795,19 @@ static void iova_magazine_free(struct iova_magazine *mag)
kfree(mag);
}
-static void
-iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
+static void iova_magazine_compact_pfns(struct iova_magazine *mag,
+ struct iova_domain *iovad,
+ unsigned long newsize)
{
unsigned long flags;
int i;
- if (!mag)
+ if (!mag || mag->size <= newsize)
return;
spin_lock_irqsave(>iova_rbtree_lock, flags);
- for (i = 0 ; i < mag->size; ++i) {
+ for (i = newsize; i < mag->size; ++i) {
struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
BUG_ON(!iova);
@@ -815,7 +816,13 @@ static void iova_magazine_free(struct iova_magazine *mag)
spin_unlock_irqrestore(>iova_rbtree_lock, flags);
- mag->size = 0;
+ mag->size = newsize;
+}
+
+static void
+iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
+{
+ iova_magazine_compact_pfns(mag, iovad, 0);
}
static bool iova_magazine_full(struct iova_magazine *mag)
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH] iommu/arm-smmu-v3: add nr_ats_masters to avoid unnecessary operations
When (smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS) is true, even if a
smmu domain does not contain any ats master, the operations of
arm_smmu_atc_inv_to_cmd() and lock protection in arm_smmu_atc_inv_domain()
are always executed. This will impact performance, especially in
multi-core and stress scenarios. For my FIO test scenario, about 8%
performance reduced.
In fact, we can use a atomic member to record how many ats masters the
smmu contains. And check that without traverse the list and check all
masters one by one in the lock protection.
Fixes: 9ce27afc0830 ("iommu/arm-smmu-v3: Add support for PCI ATS")
Signed-off-by: Zhen Lei
---
drivers/iommu/arm-smmu-v3.c | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index a9a9fabd396804a..1b370d9aca95f94 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -631,6 +631,7 @@ struct arm_smmu_domain {
struct io_pgtable_ops *pgtbl_ops;
boolnon_strict;
+ atomic_tnr_ats_masters;
enum arm_smmu_domain_stage stage;
union {
@@ -1531,7 +1532,7 @@ static int arm_smmu_atc_inv_domain(struct arm_smmu_domain
*smmu_domain,
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_master *master;
- if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
+ if (!atomic_read(_domain->nr_ats_masters))
return 0;
arm_smmu_atc_inv_to_cmd(ssid, iova, size, );
@@ -1869,6 +1870,7 @@ static int arm_smmu_enable_ats(struct arm_smmu_master
*master)
size_t stu;
struct pci_dev *pdev;
struct arm_smmu_device *smmu = master->smmu;
+ struct arm_smmu_domain *smmu_domain = master->domain;
struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
@@ -1887,12 +1889,15 @@ static int arm_smmu_enable_ats(struct arm_smmu_master
*master)
return ret;
master->ats_enabled = true;
+ atomic_inc(_domain->nr_ats_masters);
+
return 0;
}
static void arm_smmu_disable_ats(struct arm_smmu_master *master)
{
struct arm_smmu_cmdq_ent cmd;
+ struct arm_smmu_domain *smmu_domain = master->domain;
if (!master->ats_enabled || !dev_is_pci(master->dev))
return;
@@ -1901,6 +1906,7 @@ static void arm_smmu_disable_ats(struct arm_smmu_master
*master)
arm_smmu_atc_inv_master(master, );
pci_disable_ats(to_pci_dev(master->dev));
master->ats_enabled = false;
+ atomic_dec(_domain->nr_ats_masters);
}
static void arm_smmu_detach_dev(struct arm_smmu_master *master)
@@ -1915,10 +1921,10 @@ static void arm_smmu_detach_dev(struct arm_smmu_master
*master)
list_del(>domain_head);
spin_unlock_irqrestore(_domain->devices_lock, flags);
- master->domain = NULL;
arm_smmu_install_ste_for_dev(master);
arm_smmu_disable_ats(master);
+ master->domain = NULL;
}
static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
--
1.8.3
[PATCH v9 5/7] iommu/vt-d: add support for IOMMU default DMA mode build options
The default DMA mode of INTEL IOMMU is LAZY, this patch make it can be set to STRICT at build time. It can be overridden by boot option. There is no functional change. Signed-off-by: Zhen Lei --- drivers/iommu/Kconfig | 2 +- drivers/iommu/intel-iommu.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index bfbcaa24e283aad..fd297b0e0330d27 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -79,7 +79,7 @@ choice prompt "IOMMU default DMA mode" depends on IOMMU_API default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI) - default IOMMU_DEFAULT_LAZY if S390_IOMMU + default IOMMU_DEFAULT_LAZY if (INTEL_IOMMU || S390_IOMMU) default IOMMU_DEFAULT_STRICT help This option allows an IOMMU DMA mode to be chosen at build time, to diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 162b3236e72c3c8..ec5515b7831b23f 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -354,7 +354,7 @@ static int domain_detach_iommu(struct dmar_domain *domain, static int dmar_map_gfx = 1; static int dmar_forcedac; -static int intel_iommu_strict; +static int intel_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT); static int intel_iommu_superpage = 1; static int intel_iommu_sm; static int iommu_identity_mapping; -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 7/7] ia64: hide build option IOMMU_DEFAULT_PASSTHROUGH
The DMA mode PASSTHROUGH is not used on ia64. Signed-off-by: Zhen Lei --- drivers/iommu/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 70741fd73b07785..63506f1cad3d149 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -91,7 +91,7 @@ choice config IOMMU_DEFAULT_PASSTHROUGH bool "passthrough" - depends on !S390_IOMMU + depends on (!S390_IOMMU && !IA64) help In this mode, the DMA access through IOMMU without any addresses translation. That means, the wrong or illegal DMA access can not -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 3/7] s390/pci: add support for IOMMU default DMA mode build options
The default DMA mode is LAZY on s390, this patch make it can be set to
STRICT at build time. It can be overridden by boot option.
There is no functional change.
Signed-off-by: Zhen Lei
Acked-by: Sebastian Ott
---
arch/s390/pci/pci_dma.c | 2 +-
drivers/iommu/Kconfig | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 9e52d1527f71495..784ad1e0acecfb1 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -17,7 +17,7 @@
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
-static int s390_iommu_strict;
+static int s390_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
static int zpci_refresh_global(struct zpci_dev *zdev)
{
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index fe715fb295c6ed2..a8dd69d175fb3c6 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -78,6 +78,7 @@ config IOMMU_DEBUGFS
choice
prompt "IOMMU default DMA mode"
depends on IOMMU_API
+ default IOMMU_DEFAULT_LAZY if S390_IOMMU
default IOMMU_DEFAULT_STRICT
help
This option allows an IOMMU DMA mode to be chosen at build time, to
@@ -89,6 +90,7 @@ choice
config IOMMU_DEFAULT_PASSTHROUGH
bool "passthrough"
+ depends on !S390_IOMMU
help
In this mode, the DMA access through IOMMU without any addresses
translation. That means, the wrong or illegal DMA access can not
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 2/7] x86/dma: use IS_ENABLED() to simplify the code
Remove the ifdefs around CONFIG_IOMMU_DEFAULT_PASSTHROUGH to improve readablity. Signed-off-by: Zhen Lei --- arch/x86/kernel/pci-dma.c | 6 +- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index dcd272dbd0a9330..8c82b2e28a0fe2d 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -43,11 +43,7 @@ * It is also possible to disable by default in kernel config, and enable with * iommu=nopt at boot time. */ -#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH -int iommu_pass_through __read_mostly = 1; -#else -int iommu_pass_through __read_mostly; -#endif +int iommu_pass_through __read_mostly = IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH); extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 6/7] iommu/amd: add support for IOMMU default DMA mode build options
The default DMA mode of AMD IOMMU is LAZY, this patch make it can be set
to STRICT at build time. It can be overridden by boot option.
There is no functional change.
Signed-off-by: Zhen Lei
---
drivers/iommu/Kconfig | 2 +-
drivers/iommu/amd_iommu_init.c | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index fd297b0e0330d27..70741fd73b07785 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -79,7 +79,7 @@ choice
prompt "IOMMU default DMA mode"
depends on IOMMU_API
default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI)
- default IOMMU_DEFAULT_LAZY if (INTEL_IOMMU || S390_IOMMU)
+ default IOMMU_DEFAULT_LAZY if (AMD_IOMMU || INTEL_IOMMU || S390_IOMMU)
default IOMMU_DEFAULT_STRICT
help
This option allows an IOMMU DMA mode to be chosen at build time, to
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 07d84dbab564e4d..b7d5c1757425946 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -154,7 +154,7 @@ struct ivmd_header {
to handle */
LIST_HEAD(amd_iommu_unity_map);/* a list of required unity
mappings
we find in ACPI */
-bool amd_iommu_unmap_flush;/* if true, flush on every unmap */
+bool amd_iommu_unmap_flush = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT); /* if
true, flush on every unmap */
LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
system */
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 1/7] iommu: enhance IOMMU default DMA mode build options
First, add build option IOMMU_DEFAULT_{LAZY|STRICT}, so that we have the
opportunity to set {lazy|strict} mode as default at build time. Then put
the three config options in an choice, make people can only choose one of
the three at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/Kconfig | 44
drivers/iommu/iommu.c | 3 ++-
2 files changed, 38 insertions(+), 9 deletions(-)
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 83664db5221df02..fe715fb295c6ed2 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -75,16 +75,44 @@ config IOMMU_DEBUGFS
debug/iommu directory, and then populate a subdirectory with
entries as required.
-config IOMMU_DEFAULT_PASSTHROUGH
- bool "IOMMU passthrough by default"
+choice
+ prompt "IOMMU default DMA mode"
depends on IOMMU_API
-help
- Enable passthrough by default, removing the need to pass in
- iommu.passthrough=on or iommu=pt through command line. If this
- is enabled, you can still disable with iommu.passthrough=off
- or iommu=nopt depending on the architecture.
+ default IOMMU_DEFAULT_STRICT
+ help
+ This option allows an IOMMU DMA mode to be chosen at build time, to
+ override the default DMA mode of each ARCH, removing the need to
+ pass in kernel parameters through command line. You can still use
+ ARCH specific boot options to override this option again.
- If unsure, say N here.
+ If unsure, keep the default.
+
+config IOMMU_DEFAULT_PASSTHROUGH
+ bool "passthrough"
+ help
+ In this mode, the DMA access through IOMMU without any addresses
+ translation. That means, the wrong or illegal DMA access can not
+ be caught, no error information will be reported.
+
+config IOMMU_DEFAULT_LAZY
+ bool "lazy"
+ help
+ Support lazy mode, where for every IOMMU DMA unmap operation, the
+ flush operation of IOTLB and the free operation of IOVA are deferred.
+ They are only guaranteed to be done before the related IOVA will be
+ reused.
+
+config IOMMU_DEFAULT_STRICT
+ bool "strict"
+ help
+ For every IOMMU DMA unmap operation, the flush operation of IOTLB and
+ the free operation of IOVA are guaranteed to be done in the unmap
+ function.
+
+ This mode is safer than the two above, but it maybe slower in some
+ high performace scenarios.
+
+endchoice
config OF_IOMMU
def_bool y
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index f9cacce909d3ae9..05171dd0bd03aee 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -31,7 +31,8 @@
#else
static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
#endif
-static bool iommu_dma_strict __read_mostly = true;
+static bool iommu_dma_strict __read_mostly =
+ IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
struct iommu_group {
struct kobject kobj;
--
1.8.3
___
iommu mailing list
iommu@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 4/7] powernv/iommu: add support for IOMMU default DMA mode build options
The default DMA mode is PASSTHROUGH on powernv, this patch make it can be set to STRICT at build time. It can be overridden by boot option. There is no functional change. Signed-off-by: Zhen Lei --- arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++- drivers/iommu/Kconfig | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 10cc42b9e541c46..27e25e8e3a9c637 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -81,7 +81,8 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, va_end(args); } -static bool pnv_iommu_bypass_disabled __read_mostly; +static bool pnv_iommu_bypass_disabled __read_mostly = + !IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH); static bool pci_reset_phbs __read_mostly; static int __init iommu_setup(char *str) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a8dd69d175fb3c6..bfbcaa24e283aad 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -78,6 +78,7 @@ config IOMMU_DEBUGFS choice prompt "IOMMU default DMA mode" depends on IOMMU_API + default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI) default IOMMU_DEFAULT_LAZY if S390_IOMMU default IOMMU_DEFAULT_STRICT help @@ -98,6 +99,7 @@ config IOMMU_DEFAULT_PASSTHROUGH config IOMMU_DEFAULT_LAZY bool "lazy" + depends on !PPC_POWERNV help Support lazy mode, where for every IOMMU DMA unmap operation, the flush operation of IOTLB and the free operation of IOVA are deferred. -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v9 0/7] iommu: enhance IOMMU default DMA mode build options
v8--> v9 1. Fix some text editing errors v7--> v8 1. Split into multiple small patches base on ARCHs or IOMMU drivers. 2. Hide the unsupported build options on the related ARCH or IOMMU. v6 --> v7: 1. Fix some text editing errors v5 --> v6: 1. give up adding boot option iommu.dma_mode v4 --> v5: As Hanjun and Thomas Gleixner's suggestion: 1. Keep the old ARCH specific boot options no change. 2. Keep build option CONFIG_IOMMU_DEFAULT_PASSTHROUGH no change. v4: As Robin Murphy's suggestion: "It's also not necessarily obvious to the user how this interacts with IOMMU_DEFAULT_PASSTHROUGH, so if we really do go down this route, maybe it would be better to refactor the whole lot into a single selection of something like IOMMU_DEFAULT_MODE anyway." In this version, I tried to normalize the IOMMU dma mode boot options for all ARCHs. When IOMMU is enabled, there are 3 dma modes: paasthrough(bypass), lazy(mapping but defer the IOTLB invalidation), strict. But currently each ARCHs defined their private boot options, different with each other. For example, to enable/disable "passthrough", ARM64 use iommu.passthrough=1/0, X86 use iommu=pt/nopt, PPC/POWERNV use iommu=nobypass. Zhen Lei (7): iommu: enhance IOMMU default DMA mode build options x86/dma: use IS_ENABLED() to simplify the code s390/pci: add support for IOMMU default DMA mode build options powernv/iommu: add support for IOMMU default DMA mode build options iommu/vt-d: add support for IOMMU default DMA mode build options iommu/amd: add support for IOMMU default DMA mode build options ia64: hide build option IOMMU_DEFAULT_PASSTHROUGH arch/powerpc/platforms/powernv/pci-ioda.c | 3 +- arch/s390/pci/pci_dma.c | 2 +- arch/x86/kernel/pci-dma.c | 6 +--- drivers/iommu/Kconfig | 48 +-- drivers/iommu/amd_iommu_init.c| 2 +- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/iommu.c | 3 +- 7 files changed, 48 insertions(+), 18 deletions(-) -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v8 7/7] ia64: hide build option IOMMU_DEFAULT_PASSTHROUGH
The DMA mode PASSTHROUGH is not used on ia64. Signed-off-by: Zhen Lei --- drivers/iommu/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f6c030433d38048..f7400e35628dce4 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -89,7 +89,7 @@ choice config IOMMU_DEFAULT_PASSTHROUGH bool "passthrough" - depends on !S390_IOMMU + depends on (!S390_IOMMU && !IA64) help In this mode, the DMA access through IOMMU without any addresses translation. That means, the wrong or illegal DMA access can not -- 1.8.3
[PATCH v8 4/7] powernv/iommu: add support for IOMMU default DMA mode build options
The default DMA mode is PASSTHROUGH on powernv, this patch make it can be set to STRICT at build time. It can be overridden by boot option. There is no functional change. Signed-off-by: Zhen Lei --- arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++- drivers/iommu/Kconfig | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 126602b4e39972d..40208b9019be890 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -85,7 +85,8 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, va_end(args); } -static bool pnv_iommu_bypass_disabled __read_mostly; +static bool pnv_iommu_bypass_disabled __read_mostly = + !IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH); static bool pci_reset_phbs __read_mostly; static int __init iommu_setup(char *str) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9b48c2fc20e14d3..b5af859956c4fda 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -78,6 +78,7 @@ config IOMMU_DEBUGFS choice prompt "IOMMU default DMA mode" depends on IOMMU_API + default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI) default IOMMU_DEFAULT_LAZY if S390_IOMMU default IOMMU_DEFAULT_STRICT help @@ -98,6 +99,7 @@ config IOMMU_DEFAULT_PASSTHROUGH config IOMMU_DEFAULT_LAZY bool "lazy" + depends on !PPC_POWERNV help Support lazy mode, where for every IOMMU DMA unmap operation, the flush operation of IOTLB and the free operation of IOVA are deferred. -- 1.8.3
[PATCH v8 2/7] x86/dma: use IS_ENABLED() to simplify the code
This patch removes the ifdefs around CONFIG_IOMMU_DEFAULT_PASSTHROUGH to improve readablity. Signed-off-by: Zhen Lei --- arch/x86/kernel/pci-dma.c | 7 ++- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index dcd272dbd0a9330..9f2b19c35a060df 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c @@ -43,11 +43,8 @@ * It is also possible to disable by default in kernel config, and enable with * iommu=nopt at boot time. */ -#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH -int iommu_pass_through __read_mostly = 1; -#else -int iommu_pass_through __read_mostly; -#endif +int iommu_pass_through __read_mostly = + IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH); extern struct iommu_table_entry __iommu_table[], __iommu_table_end[]; -- 1.8.3
[PATCH v8 6/7] iommu/amd: add support for IOMMU default DMA mode build options
The default DMA mode of AMD IOMMU is LAZY, this patch make it can be set
to STRICT at build time. It can be overridden by boot option.
There is no functional change.
Signed-off-by: Zhen Lei
---
drivers/iommu/Kconfig | 2 +-
drivers/iommu/amd_iommu_init.c | 3 ++-
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index af580274b7c5270..f6c030433d38048 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -79,7 +79,7 @@ choice
prompt "IOMMU default DMA mode"
depends on IOMMU_API
default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI)
- default IOMMU_DEFAULT_LAZY if (INTEL_IOMMU || S390_IOMMU)
+ default IOMMU_DEFAULT_LAZY if (AMD_IOMMU || INTEL_IOMMU || S390_IOMMU)
default IOMMU_DEFAULT_STRICT
help
This option allows IOMMU DMA mode to be chose at build time, to
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index f977df90d2a4912..6b0bfa43f6faa32 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -166,7 +166,8 @@ struct ivmd_header {
to handle */
LIST_HEAD(amd_iommu_unity_map);/* a list of required unity
mappings
we find in ACPI */
-bool amd_iommu_unmap_flush;/* if true, flush on every unmap */
+bool amd_iommu_unmap_flush = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
+ /* if true, flush on every unmap */
LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
system */
--
1.8.3
[PATCH v8 0/7] iommu: enhance IOMMU default DMA mode build options
v7--> v8 1. Split into multiple small patches base on ARCHs or IOMMU drivers. 2. Hide the unsupported build options on the related ARCH or IOMMU. v6 --> v7: 1. Fix some text editing errors v5 --> v6: 1. give up adding boot option iommu.dma_mode v4 --> v5: As Hanjun and Thomas Gleixner's suggestion: 1. Keep the old ARCH specific boot options no change. 2. Keep build option CONFIG_IOMMU_DEFAULT_PASSTHROUGH no change. v4: As Robin Murphy's suggestion: "It's also not necessarily obvious to the user how this interacts with IOMMU_DEFAULT_PASSTHROUGH, so if we really do go down this route, maybe it would be better to refactor the whole lot into a single selection of something like IOMMU_DEFAULT_MODE anyway." In this version, I tried to normalize the IOMMU dma mode boot options for all ARCHs. When IOMMU is enabled, there are 3 dma modes: paasthrough(bypass), lazy(mapping but defer the IOTLB invalidation), strict. But currently each ARCHs defined their private boot options, different with each other. For example, to enable/disable "passthrough", ARM64 use iommu.passthrough=1/0, X86 use iommu=pt/nopt, PPC/POWERNV use iommu=nobypass. Zhen Lei (7): iommu: enhance IOMMU default DMA mode build options x86/dma: use IS_ENABLED() to simplify the code s390/pci: add support for IOMMU default DMA mode build options powernv/iommu: add support for IOMMU default DMA mode build options iommu/vt-d: add support for IOMMU default DMA mode build options iommu/amd: add support for IOMMU default DMA mode build options ia64: hide build option IOMMU_DEFAULT_PASSTHROUGH arch/powerpc/platforms/powernv/pci-ioda.c | 3 +- arch/s390/pci/pci_dma.c | 2 +- arch/x86/kernel/pci-dma.c | 7 ++--- drivers/iommu/Kconfig | 46 ++- drivers/iommu/amd_iommu_init.c| 3 +- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/iommu.c | 3 +- 7 files changed, 49 insertions(+), 17 deletions(-) -- 1.8.3
[PATCH v8 5/7] iommu/vt-d: add support for IOMMU default DMA mode build options
The default DMA mode of INTEL IOMMU is LAZY, this patch make it can be set to STRICT at build time. It can be overridden by boot option. There is no functional change. Signed-off-by: Zhen Lei --- drivers/iommu/Kconfig | 2 +- drivers/iommu/intel-iommu.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index b5af859956c4fda..af580274b7c5270 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -79,7 +79,7 @@ choice prompt "IOMMU default DMA mode" depends on IOMMU_API default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI) - default IOMMU_DEFAULT_LAZY if S390_IOMMU + default IOMMU_DEFAULT_LAZY if (INTEL_IOMMU || S390_IOMMU) default IOMMU_DEFAULT_STRICT help This option allows IOMMU DMA mode to be chose at build time, to diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a209199f3af6460..50d74ea0acdbdca 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -362,7 +362,7 @@ static int domain_detach_iommu(struct dmar_domain *domain, static int dmar_map_gfx = 1; static int dmar_forcedac; -static int intel_iommu_strict; +static int intel_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT); static int intel_iommu_superpage = 1; static int intel_iommu_sm; static int iommu_identity_mapping; -- 1.8.3
[PATCH v8 3/7] s390/pci: add support for IOMMU default DMA mode build options
The default DMA mode is LAZY on s390, this patch make it can be set to
STRICT at build time. It can be overridden by boot option.
There is no functional change.
Signed-off-by: Zhen Lei
---
arch/s390/pci/pci_dma.c | 2 +-
drivers/iommu/Kconfig | 2 ++
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 9e52d1527f71495..784ad1e0acecfb1 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -17,7 +17,7 @@
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
-static int s390_iommu_strict;
+static int s390_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
static int zpci_refresh_global(struct zpci_dev *zdev)
{
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index d6a1a45f80ffbf5..9b48c2fc20e14d3 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -78,6 +78,7 @@ config IOMMU_DEBUGFS
choice
prompt "IOMMU default DMA mode"
depends on IOMMU_API
+ default IOMMU_DEFAULT_LAZY if S390_IOMMU
default IOMMU_DEFAULT_STRICT
help
This option allows IOMMU DMA mode to be chose at build time, to
@@ -87,6 +88,7 @@ choice
config IOMMU_DEFAULT_PASSTHROUGH
bool "passthrough"
+ depends on !S390_IOMMU
help
In this mode, the DMA access through IOMMU without any addresses
translation. That means, the wrong or illegal DMA access can not
--
1.8.3
[PATCH v8 1/7] iommu: enhance IOMMU default DMA mode build options
First, add build option IOMMU_DEFAULT_{LAZY|STRICT}, so that we have the
opportunity to set {lazy|strict} mode as default at build time. Then put
the three config options in an choice, make people can only choose one of
the three at a time.
Signed-off-by: Zhen Lei
---
drivers/iommu/Kconfig | 42 +++---
drivers/iommu/iommu.c | 3 ++-
2 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 83664db5221df02..d6a1a45f80ffbf5 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -75,17 +75,45 @@ config IOMMU_DEBUGFS
debug/iommu directory, and then populate a subdirectory with
entries as required.
-config IOMMU_DEFAULT_PASSTHROUGH
- bool "IOMMU passthrough by default"
+choice
+ prompt "IOMMU default DMA mode"
depends on IOMMU_API
-help
- Enable passthrough by default, removing the need to pass in
- iommu.passthrough=on or iommu=pt through command line. If this
- is enabled, you can still disable with iommu.passthrough=off
- or iommu=nopt depending on the architecture.
+ default IOMMU_DEFAULT_STRICT
+ help
+ This option allows IOMMU DMA mode to be chose at build time, to
+ override the default DMA mode of each ARCHs, removing the need to
+ pass in kernel parameters through command line. You can still use
+ ARCHs specific boot options to override this option again.
+
+config IOMMU_DEFAULT_PASSTHROUGH
+ bool "passthrough"
+ help
+ In this mode, the DMA access through IOMMU without any addresses
+ translation. That means, the wrong or illegal DMA access can not
+ be caught, no error information will be reported.
If unsure, say N here.
+config IOMMU_DEFAULT_LAZY
+ bool "lazy"
+ help
+ Support lazy mode, where for every IOMMU DMA unmap operation, the
+ flush operation of IOTLB and the free operation of IOVA are deferred.
+ They are only guaranteed to be done before the related IOVA will be
+ reused.
+
+config IOMMU_DEFAULT_STRICT
+ bool "strict"
+ help
+ For every IOMMU DMA unmap operation, the flush operation of IOTLB and
+ the free operation of IOVA are guaranteed to be done in the unmap
+ function.
+
+ This mode is safer than the two above, but it maybe slower in some
+ high performace scenarios.
+
+endchoice
+
config OF_IOMMU
def_bool y
depends on OF && IOMMU_API
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index 67ee6623f9b2a4d..56bce221285b15f 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -43,7 +43,8 @@
#else
static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
#endif
-static bool iommu_dma_strict __read_mostly = true;
+static bool iommu_dma_strict __read_mostly =
+ IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
struct iommu_group {
struct kobject kobj;
--
1.8.3
[PATCH v7 1/1] iommu: enhance IOMMU dma mode build options
First, add build option IOMMU_DEFAULT_{LAZY|STRICT}, so that we have the
opportunity to set {lazy|strict} mode as default at build time. Then put
the three config options in an choice, make people can only choose one of
the three at a time.
The default IOMMU dma modes on each ARCHs have no change.
Signed-off-by: Zhen Lei
---
arch/ia64/kernel/pci-dma.c| 2 +-
arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++-
arch/s390/pci/pci_dma.c | 2 +-
arch/x86/kernel/pci-dma.c | 7 ++---
drivers/iommu/Kconfig | 44 ++-
drivers/iommu/amd_iommu_init.c| 3 ++-
drivers/iommu/intel-iommu.c | 2 +-
drivers/iommu/iommu.c | 3 ++-
8 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index fe988c49f01ce6a..655511dbf3c3b34 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -22,7 +22,7 @@
int force_iommu __read_mostly;
#endif
-int iommu_pass_through;
+int iommu_pass_through = IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
static int __init pci_iommu_init(void)
{
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c
b/arch/powerpc/platforms/powernv/pci-ioda.c
index 3ead4c237ed0ec9..383e082a9bb985c 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -85,7 +85,8 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char
*level,
va_end(args);
}
-static bool pnv_iommu_bypass_disabled __read_mostly;
+static bool pnv_iommu_bypass_disabled __read_mostly =
+ !IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
static bool pci_reset_phbs __read_mostly;
static int __init iommu_setup(char *str)
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 9e52d1527f71495..784ad1e0acecfb1 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -17,7 +17,7 @@
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
-static int s390_iommu_strict;
+static int s390_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
static int zpci_refresh_global(struct zpci_dev *zdev)
{
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index d460998ae828514..fb2bab42a0a3173 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -43,11 +43,8 @@
* It is also possible to disable by default in kernel config, and enable with
* iommu=nopt at boot time.
*/
-#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
-int iommu_pass_through __read_mostly = 1;
-#else
-int iommu_pass_through __read_mostly;
-#endif
+int iommu_pass_through __read_mostly =
+ IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 6f07f3b21816c64..8a1f1793cde76b4 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -74,17 +74,47 @@ config IOMMU_DEBUGFS
debug/iommu directory, and then populate a subdirectory with
entries as required.
-config IOMMU_DEFAULT_PASSTHROUGH
- bool "IOMMU passthrough by default"
+choice
+ prompt "IOMMU default DMA mode"
depends on IOMMU_API
-help
- Enable passthrough by default, removing the need to pass in
- iommu.passthrough=on or iommu=pt through command line. If this
- is enabled, you can still disable with iommu.passthrough=off
- or iommu=nopt depending on the architecture.
+ default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI)
+ default IOMMU_DEFAULT_LAZY if (AMD_IOMMU || INTEL_IOMMU || S390_IOMMU)
+ default IOMMU_DEFAULT_STRICT
+ help
+ This option allows IOMMU DMA mode to be chose at build time, to
+ override the default DMA mode of each ARCHs, removing the need to
+ pass in kernel parameters through command line. You can still use
+ ARCHs specific boot options to override this option again.
+
+config IOMMU_DEFAULT_PASSTHROUGH
+ bool "passthrough"
+ help
+ In this mode, the DMA access through IOMMU without any addresses
+ translation. That means, the wrong or illegal DMA access can not
+ be caught, no error information will be reported.
If unsure, say N here.
+config IOMMU_DEFAULT_LAZY
+ bool "lazy"
+ help
+ Support lazy mode, where for every IOMMU DMA unmap operation, the
+ flush operation of IOTLB and the free operation of IOVA are deferred.
+ They are only guaranteed to be done before the related IOVA will be
+ reused.
+
+config IOMMU_DEFAULT_STRICT
+ bool "strict"
+ help
+ For every IOMMU DMA unmap operation, the flush operation of IOTLB and
+
[PATCH v7 0/1] iommu: enhance IOMMU dma mode build options
v6 --> v7: 1. Fix some text editing errors v5 --> v6: 1. give up adding boot option iommu.dma_mode v4 --> v5: As Hanjun and Thomas Gleixner's suggestion: 1. Keep the old ARCH specific boot options no change. 2. Keep build option CONFIG_IOMMU_DEFAULT_PASSTHROUGH no change. v4: As Robin Murphy's suggestion: "It's also not necessarily obvious to the user how this interacts with IOMMU_DEFAULT_PASSTHROUGH, so if we really do go down this route, maybe it would be better to refactor the whole lot into a single selection of something like IOMMU_DEFAULT_MODE anyway." In this version, I tried to normalize the IOMMU dma mode boot options for all ARCHs. When IOMMU is enabled, there are 3 dma modes: paasthrough(bypass), lazy(mapping but defer the IOTLB invalidation), strict. But currently each ARCHs defined their private boot options, different with each other. For example, to enable/disable "passthrough", ARM64 use iommu.passthrough=1/0, X86 use iommu=pt/nopt, PPC/POWERNV use iommu=nobypass. Zhen Lei (1): iommu: enhance IOMMU dma mode build options arch/ia64/kernel/pci-dma.c| 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++- arch/s390/pci/pci_dma.c | 2 +- arch/x86/kernel/pci-dma.c | 7 ++--- drivers/iommu/Kconfig | 44 ++- drivers/iommu/amd_iommu_init.c| 3 ++- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/iommu.c | 3 ++- 8 files changed, 48 insertions(+), 18 deletions(-) -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v6 1/1] iommu: enhance IOMMU dma mode build options
First, add build option IOMMU_DEFAULT_{LAZY|STRICT}, so that we have the
opportunity to set {lazy|strict} mode as default at build time. Then put
the three config options in an choice, make people can only choose one of
the three at a time.
The default IOMMU dma modes on each ARCHs have no change.
Signed-off-by: Zhen Lei
---
arch/ia64/kernel/pci-dma.c| 2 +-
arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++-
arch/s390/pci/pci_dma.c | 2 +-
arch/x86/kernel/pci-dma.c | 7 ++---
drivers/iommu/Kconfig | 44 ++-
drivers/iommu/amd_iommu_init.c| 3 ++-
drivers/iommu/intel-iommu.c | 2 +-
drivers/iommu/iommu.c | 3 ++-
8 files changed, 48 insertions(+), 18 deletions(-)
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index fe988c49f01ce6a..655511dbf3c3b34 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -22,7 +22,7 @@
int force_iommu __read_mostly;
#endif
-int iommu_pass_through;
+int iommu_pass_through = IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
static int __init pci_iommu_init(void)
{
diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c
b/arch/powerpc/platforms/powernv/pci-ioda.c
index 3ead4c237ed0ec9..383e082a9bb985c 100644
--- a/arch/powerpc/platforms/powernv/pci-ioda.c
+++ b/arch/powerpc/platforms/powernv/pci-ioda.c
@@ -85,7 +85,8 @@ void pe_level_printk(const struct pnv_ioda_pe *pe, const char
*level,
va_end(args);
}
-static bool pnv_iommu_bypass_disabled __read_mostly;
+static bool pnv_iommu_bypass_disabled __read_mostly =
+ !IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
static bool pci_reset_phbs __read_mostly;
static int __init iommu_setup(char *str)
diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index 9e52d1527f71495..784ad1e0acecfb1 100644
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -17,7 +17,7 @@
static struct kmem_cache *dma_region_table_cache;
static struct kmem_cache *dma_page_table_cache;
-static int s390_iommu_strict;
+static int s390_iommu_strict = IS_ENABLED(CONFIG_IOMMU_DEFAULT_STRICT);
static int zpci_refresh_global(struct zpci_dev *zdev)
{
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index d460998ae828514..fb2bab42a0a3173 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -43,11 +43,8 @@
* It is also possible to disable by default in kernel config, and enable with
* iommu=nopt at boot time.
*/
-#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
-int iommu_pass_through __read_mostly = 1;
-#else
-int iommu_pass_through __read_mostly;
-#endif
+int iommu_pass_through __read_mostly =
+ IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH);
extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 6f07f3b21816c64..8a1f1793cde76b4 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -74,17 +74,47 @@ config IOMMU_DEBUGFS
debug/iommu directory, and then populate a subdirectory with
entries as required.
-config IOMMU_DEFAULT_PASSTHROUGH
- bool "IOMMU passthrough by default"
+choice
+ prompt "IOMMU dma mode"
depends on IOMMU_API
-help
- Enable passthrough by default, removing the need to pass in
- iommu.passthrough=on or iommu=pt through command line. If this
- is enabled, you can still disable with iommu.passthrough=off
- or iommu=nopt depending on the architecture.
+ default IOMMU_DEFAULT_PASSTHROUGH if (PPC_POWERNV && PCI)
+ default IOMMU_DEFAULT_LAZY if (AMD_IOMMU || INTEL_IOMMU || S390_IOMMU)
+ default IOMMU_DEFAULT_STRICT
+ help
+ This option allows IOMMU dma mode to be chose at build time, to
+ override the default dma mode of each ARCHs, removing the need to
+ pass in kernel parameters through command line. You can still use
+ ARCHs specific boot options to override this option again.
+
+config IOMMU_DEFAULT_PASSTHROUGH
+ bool "passthrough"
+ help
+ In this mode, the dma access through IOMMU without any addresses
+ transformation. That means, the wrong or illegal dma access can not
+ be caught, no error information will be reported.
If unsure, say N here.
+config IOMMU_DEFAULT_LAZY
+ bool "lazy"
+ help
+ Support lazy mode, where for every IOMMU DMA unmap operation, the
+ flush operation of IOTLB and the free operation of IOVA are deferred.
+ They are only guaranteed to be done before the related IOVA will be
+ reused.
+
+config IOMMU_DEFAULT_STRICT
+ bool "strict"
+ help
+ For every IOMMU DMA unmap operation, the flush operation of IOTLB and
+ the fre
[PATCH v6 0/1] iommu: enhance IOMMU dma mode build options
v5 --> v6: 1. give up adding boot option iommu.dma_mode v4 --> v5: As Hanjun and Thomas Gleixner's suggestion: 1. Keep the old ARCH specific boot options no change. 2. Keep build option CONFIG_IOMMU_DEFAULT_PASSTHROUGH no change. v4: As Robin Murphy's suggestion: "It's also not necessarily obvious to the user how this interacts with IOMMU_DEFAULT_PASSTHROUGH, so if we really do go down this route, maybe it would be better to refactor the whole lot into a single selection of something like IOMMU_DEFAULT_MODE anyway." In this version, I tried to normalize the IOMMU dma mode boot options for all ARCHs. When IOMMU is enabled, there are 3 dma modes: paasthrough(bypass), lazy(mapping but defer the IOTLB invalidation), strict. But currently each ARCHs defined their private boot options, different with each other. For example, to enable/disable "passthrough", ARM64 use iommu.passthrough=1/0, X86 use iommu=pt/nopt, PPC/POWERNV use iommu=nobypass. Zhen Lei (1): iommu: enhance IOMMU dma mode build options arch/ia64/kernel/pci-dma.c| 2 +- arch/powerpc/platforms/powernv/pci-ioda.c | 3 ++- arch/s390/pci/pci_dma.c | 2 +- arch/x86/kernel/pci-dma.c | 7 ++--- drivers/iommu/Kconfig | 44 ++- drivers/iommu/amd_iommu_init.c| 3 ++- drivers/iommu/intel-iommu.c | 2 +- drivers/iommu/iommu.c | 3 ++- 8 files changed, 48 insertions(+), 18 deletions(-) -- 1.8.3 ___ iommu mailing list iommu@lists.linux-foundation.org https://lists.linuxfoundation.org/mailman/listinfo/iommu
[PATCH v5 6/6] x86/iommu: add support for generic boot option iommu.dma_mode
The following equivalence or replacement relationship exists:
iommu=pt <--> iommu.dma_mode=passthrough.
iommu=nopt can be replaced with iommu.dma_mode=lazy.
intel_iommu=strict <--> iommu.dma_mode=strict.
amd_iommu=fullflush <--> iommu.dma_mode=strict.
Signed-off-by: Zhen Lei
---
Documentation/admin-guide/kernel-parameters.txt | 6 ++---
arch/ia64/include/asm/iommu.h | 2 --
arch/ia64/kernel/pci-dma.c | 2 --
arch/x86/include/asm/iommu.h| 1 -
arch/x86/kernel/pci-dma.c | 35 -
drivers/iommu/Kconfig | 2 +-
drivers/iommu/amd_iommu.c | 10 +++
drivers/iommu/amd_iommu_init.c | 4 +--
drivers/iommu/amd_iommu_types.h | 6 -
drivers/iommu/intel-iommu.c | 9 +++
10 files changed, 31 insertions(+), 46 deletions(-)
diff --git a/Documentation/admin-guide/kernel-parameters.txt
b/Documentation/admin-guide/kernel-parameters.txt
index 176f96032d9d62a..ca6edc529d6a614 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -1815,9 +1815,9 @@
options(such as CONFIG_IOMMU_DEFAULT_PASSTHROUGH) to
choose which mode to be used.
Note: For historical reasons, ARM64/S390/PPC/X86 have
- their specific options. Currently, only ARM64/S390/PPC
- support this boot option, and hope other ARCHs to use
- this as generic boot option.
+ their specific options, but strongly recommended switch
+ to use this one, the new ARCHs should use this generic
+ boot option.
passthrough
Configure DMA to bypass the IOMMU by default.
lazy
diff --git a/arch/ia64/include/asm/iommu.h b/arch/ia64/include/asm/iommu.h
index 7429a72f3f92199..92aceef63710861 100644
--- a/arch/ia64/include/asm/iommu.h
+++ b/arch/ia64/include/asm/iommu.h
@@ -8,10 +8,8 @@
extern void no_iommu_init(void);
#ifdef CONFIG_INTEL_IOMMU
extern int force_iommu, no_iommu;
-extern int iommu_pass_through;
extern int iommu_detected;
#else
-#define iommu_pass_through (0)
#define no_iommu (1)
#define iommu_detected (0)
#endif
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index fe988c49f01ce6a..f5d49cd3fbb01a9 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -22,8 +22,6 @@
int force_iommu __read_mostly;
#endif
-int iommu_pass_through;
-
static int __init pci_iommu_init(void)
{
if (iommu_detected)
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index baedab8ac5385f7..b91623d521d9f0f 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -4,7 +4,6 @@
extern int force_iommu, no_iommu;
extern int iommu_detected;
-extern int iommu_pass_through;
/* 10 seconds */
#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index d460998ae828514..fc64928e47cb860 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -6,6 +6,7 @@
#include
#include
#include
+#include
#include
#include
@@ -34,21 +35,6 @@
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;
-/*
- * This variable becomes 1 if iommu=pt is passed on the kernel command line.
- * If this variable is 1, IOMMU implementations do no DMA translation for
- * devices and allow every device to access to whole physical memory. This is
- * useful if a user wants to use an IOMMU only for KVM device assignment to
- * guests and not for driver dma translation.
- * It is also possible to disable by default in kernel config, and enable with
- * iommu=nopt at boot time.
- */
-#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
-int iommu_pass_through __read_mostly = 1;
-#else
-int iommu_pass_through __read_mostly;
-#endif
-
extern struct iommu_table_entry __iommu_table[], __iommu_table_end[];
/* Dummy device used for NULL arguments (normally ISA). */
@@ -139,10 +125,23 @@ static __init int iommu_setup(char *p)
if (!strncmp(p, "soft", 4))
swiotlb = 1;
#endif
+
+ /*
+* IOMMU implementations do no DMA translation for devices and
+* allow every device to access to whole physical memory. This
+* is useful if a user wants to use an IOMMU only for KVM
+* device assignment to guests and not for driver dma
+* translation.
+*/
if (!strncmp(p, "pt", 2))
- iommu_pass_through = 1;
-
