Currently, tcf_block doesn't use any synchronization mechanisms to protect
code that manages lifetime of its chains. block->chain_list and multiple
variables in tcf_chain that control its lifetime assume external
synchronization provided by global rtnl lock. Converting chain reference
counting to atomic reference counters is not possible because cls API uses
multiple counters and flags to control chain lifetime, so all of them must
be synchronized in chain get/put code.
Use single per-block lock to protect block data and manage lifetime of all
chains on the block. Always take block->lock when accessing chain_list.
Chain get and put modify chain lifetime-management data and parent block's
chain_list, so take the lock in these functions. Verify block->lock state
with assertions in functions that expect to be called with the lock taken
and are called from multiple places. Take block->lock when accessing
filter_chain_list.
block->lock is a spinlock which means blocking functions like classifier
ops callbacks cannot be called while holding it. Rearrange chain get and
put functions code to only access protected chain data while holding block
lock and move blocking calls outside critical section:
- Check if chain was explicitly created inside put function while holding
block lock. Add additional argument to __tcf_chain_put() to only put
explicitly created chain.
- Rearrange code to only access chain reference counter and chain action
reference counter while holding block lock.
- Split tcf_chain_destroy() helper to two functions: one that requires
block->lock, and another one that needs to call sleeping functions and
can be executed after lock is released. First helper is used to detach
chain from block and make it inaccessible for concurrent users, second
actually deallocates chain memory (and parent block, if applicable).
Signed-off-by: Vlad Buslov <vla...@mellanox.com>
Acked-by: Jiri Pirko <j...@mellanox.com>
---
Changes from V1 to V2:
- Use lockdep_assert_held() instead of spin_is_locked() for assertion.
- Use 'free_block' argument in tcf_chain_destroy() instead of checking
block's reference count and chain_list for second time.
include/net/sch_generic.h | 4 ++
net/sched/cls_api.c | 94 ++++++++++++++++++++++++++++++++++++++---------
2 files changed, 81 insertions(+), 17 deletions(-)
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 80fc06468c79..61efc4fd469a 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -359,6 +359,10 @@ struct tcf_chain {
};
struct tcf_block {
+ /* Lock protects tcf_block and lifetime-management data of chains
+ * attached to the block (refcnt, action_refcnt, explicitly_created).
+ */
+ spinlock_t lock;
struct list_head chain_list;
u32 index; /* block index for shared blocks */
refcount_t refcnt;
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 08f245abf04a..a5aed85f0e5a 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -217,6 +217,9 @@ static void tcf_proto_destroy(struct tcf_proto *tp,
tc_queue_proto_work(&tp->work);
}
+#define ASSERT_BLOCK_LOCKED(block) \
+ lockdep_assert_held(&(block)->lock)
+
struct tcf_filter_chain_list_item {
struct list_head list;
tcf_chain_head_change_t *chain_head_change;
@@ -228,7 +231,9 @@ static struct tcf_chain *tcf_chain_create(struct tcf_block
*block,
{
struct tcf_chain *chain;
- chain = kzalloc(sizeof(*chain), GFP_KERNEL);
+ ASSERT_BLOCK_LOCKED(block);
+
+ chain = kzalloc(sizeof(*chain), GFP_ATOMIC);
if (!chain)
return NULL;
list_add_tail(&chain->list, &block->chain_list);
@@ -259,25 +264,45 @@ static void tcf_chain0_head_change(struct tcf_chain
*chain,
tcf_chain_head_change_item(item, tp_head);
}
-static void tcf_chain_destroy(struct tcf_chain *chain)
+/* Returns true if block can be safely freed. */
+
+static bool tcf_chain_detach(struct tcf_chain *chain)
{
struct tcf_block *block = chain->block;
+ ASSERT_BLOCK_LOCKED(block);
+
list_del(&chain->list);
if (!chain->index)
block->chain0.chain = NULL;
+
+ if (list_empty(&block->chain_list) &&
+ refcount_read(&block->refcnt) == 0)
+ return true;
+
+ return false;
+}
+
+static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block)
+{
+ struct tcf_block *block = chain->block;
+
kfree(chain);
- if (list_empty(&block->chain_list) && !refcount_read(&block->refcnt))
+ if (free_block)
kfree_rcu(block, rcu);
}
static void tcf_chain_hold(struct tcf_chain *chain)
{
+ ASSERT_BLOCK_LOCKED(chain->block);
+
++chain->refcnt;
}
static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain)
{
+ ASSERT_BLOCK_LOCKED(chain->block);
+
/* In case all the references are action references, this
* chain should not be shown to the user.
*/
@@ -289,6 +314,8 @@ static struct tcf_chain *tcf_chain_lookup(struct tcf_block
*block,
{
struct tcf_chain *chain;
+ ASSERT_BLOCK_LOCKED(block);
+
list_for_each_entry(chain, &block->chain_list, list) {
if (chain->index == chain_index)
return chain;
@@ -303,31 +330,40 @@ static struct tcf_chain *__tcf_chain_get(struct tcf_block
*block,
u32 chain_index, bool create,
bool by_act)
{
- struct tcf_chain *chain = tcf_chain_lookup(block, chain_index);
+ struct tcf_chain *chain = NULL;
+ bool is_first_reference;
+ spin_lock(&block->lock);
+ chain = tcf_chain_lookup(block, chain_index);
if (chain) {
tcf_chain_hold(chain);
} else {
if (!create)
- return NULL;
+ goto errout;
chain = tcf_chain_create(block, chain_index);
if (!chain)
- return NULL;
+ goto errout;
}
if (by_act)
++chain->action_refcnt;
+ is_first_reference = chain->refcnt - chain->action_refcnt == 1;
+ spin_unlock(&block->lock);
/* Send notification only in case we got the first
* non-action reference. Until then, the chain acts only as
* a placeholder for actions pointing to it and user ought
* not know about them.
*/
- if (chain->refcnt - chain->action_refcnt == 1 && !by_act)
+ if (is_first_reference && !by_act)
tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL,
RTM_NEWCHAIN, false);
return chain;
+
+errout:
+ spin_unlock(&block->lock);
+ return chain;
}
static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32
chain_index,
@@ -344,37 +380,59 @@ EXPORT_SYMBOL(tcf_chain_get_by_act);
static void tc_chain_tmplt_del(struct tcf_chain *chain);
-static void __tcf_chain_put(struct tcf_chain *chain, bool by_act)
+static void __tcf_chain_put(struct tcf_chain *chain, bool by_act,
+ bool explicitly_created)
{
+ struct tcf_block *block = chain->block;
+ bool is_last, free_block = false;
+ unsigned int refcnt;
+
+ spin_lock(&block->lock);
+ if (explicitly_created) {
+ if (!chain->explicitly_created) {
+ spin_unlock(&block->lock);
+ return;
+ }
+ chain->explicitly_created = false;
+ }
+
if (by_act)
chain->action_refcnt--;
- chain->refcnt--;
+
+ /* tc_chain_notify_delete can't be called while holding block lock.
+ * However, when block is unlocked chain can be changed concurrently, so
+ * save these to temporary variables.
+ */
+ refcnt = --chain->refcnt;
+ is_last = refcnt - chain->action_refcnt == 0;
+ if (refcnt == 0)
+ free_block = tcf_chain_detach(chain);
+ spin_unlock(&block->lock);
/* The last dropped non-action reference will trigger notification. */
- if (chain->refcnt - chain->action_refcnt == 0 && !by_act)
+ if (is_last && !by_act)
tc_chain_notify(chain, NULL, 0, 0, RTM_DELCHAIN, false);
- if (chain->refcnt == 0) {
+ if (refcnt == 0) {
tc_chain_tmplt_del(chain);
- tcf_chain_destroy(chain);
+ tcf_chain_destroy(chain, free_block);
}
}
static void tcf_chain_put(struct tcf_chain *chain)
{
- __tcf_chain_put(chain, false);
+ __tcf_chain_put(chain, false, false);
}
void tcf_chain_put_by_act(struct tcf_chain *chain)
{
- __tcf_chain_put(chain, true);
+ __tcf_chain_put(chain, true, false);
}
EXPORT_SYMBOL(tcf_chain_put_by_act);
static void tcf_chain_put_explicitly_created(struct tcf_chain *chain)
{
- if (chain->explicitly_created)
- tcf_chain_put(chain);
+ __tcf_chain_put(chain, false, true);
}
static void tcf_chain_flush(struct tcf_chain *chain)
@@ -787,6 +845,7 @@ static struct tcf_block *tcf_block_create(struct net *net,
struct Qdisc *q,
NL_SET_ERR_MSG(extack, "Memory allocation for block failed");
return ERR_PTR(-ENOMEM);
}
+ spin_lock_init(&block->lock);
INIT_LIST_HEAD(&block->chain_list);
INIT_LIST_HEAD(&block->cb_list);
INIT_LIST_HEAD(&block->owner_list);
@@ -850,7 +909,7 @@ static void tcf_block_put_all_chains(struct tcf_block
*block)
static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q,
struct tcf_block_ext_info *ei)
{
- if (refcount_dec_and_test(&block->refcnt)) {
+ if (refcount_dec_and_lock(&block->refcnt, &block->lock)) {
/* Flushing/putting all chains will cause the block to be
* deallocated when last chain is freed. However, if chain_list
* is empty, block has to be manually deallocated. After block
@@ -859,6 +918,7 @@ static void __tcf_block_put(struct tcf_block *block, struct
Qdisc *q,
*/
bool free_block = list_empty(&block->chain_list);
+ spin_unlock(&block->lock);
if (tcf_block_shared(block))
tcf_block_remove(block, block->net);
if (!free_block)
--
2.7.5
