2005-04-16 18:20:36 -04:00
|
|
|
/*
|
|
|
|
* net/sched/sch_generic.c Generic packet scheduler routines.
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
|
|
* Jamal Hadi Salim, <hadi@cyberus.ca> 990601
|
|
|
|
* - Ingress support
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/rtnetlink.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <net/pkt_sched.h>
|
|
|
|
|
|
|
|
/* Main transmission queue. */
|
|
|
|
|
2007-04-16 20:02:10 -04:00
|
|
|
/* Modifications to data participating in scheduling must be protected with
|
|
|
|
* dev->queue_lock spinlock.
|
|
|
|
*
|
|
|
|
* The idea is the following:
|
|
|
|
* - enqueue, dequeue are serialized via top level device
|
|
|
|
* spinlock dev->queue_lock.
|
2007-04-16 20:07:08 -04:00
|
|
|
* - ingress filtering is serialized via top level device
|
|
|
|
* spinlock dev->ingress_lock.
|
2007-04-16 20:02:10 -04:00
|
|
|
* - updates to tree and tree walking are only done under the rtnl mutex.
|
2005-04-16 18:20:36 -04:00
|
|
|
*/
|
|
|
|
|
|
|
|
void qdisc_lock_tree(struct net_device *dev)
|
2008-01-02 00:58:02 -05:00
|
|
|
__acquires(dev->queue_lock)
|
|
|
|
__acquires(dev->ingress_lock)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
spin_lock_bh(&dev->queue_lock);
|
2007-04-16 20:07:08 -04:00
|
|
|
spin_lock(&dev->ingress_lock);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(qdisc_lock_tree);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
void qdisc_unlock_tree(struct net_device *dev)
|
2008-01-02 00:58:02 -05:00
|
|
|
__releases(dev->ingress_lock)
|
|
|
|
__releases(dev->queue_lock)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
2007-04-16 20:07:08 -04:00
|
|
|
spin_unlock(&dev->ingress_lock);
|
2005-04-16 18:20:36 -04:00
|
|
|
spin_unlock_bh(&dev->queue_lock);
|
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(qdisc_unlock_tree);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2007-06-10 20:31:24 -04:00
|
|
|
static inline int qdisc_qlen(struct Qdisc *q)
|
|
|
|
{
|
|
|
|
return q->q.qlen;
|
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
static inline int dev_requeue_skb(struct sk_buff *skb, struct net_device *dev,
|
|
|
|
struct Qdisc *q)
|
2007-06-10 20:31:24 -04:00
|
|
|
{
|
|
|
|
if (unlikely(skb->next))
|
|
|
|
dev->gso_skb = skb;
|
|
|
|
else
|
|
|
|
q->ops->requeue(skb, q);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
|
2007-06-10 20:31:24 -04:00
|
|
|
netif_schedule(dev);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
static inline struct sk_buff *dev_dequeue_skb(struct net_device *dev,
|
|
|
|
struct Qdisc *q)
|
2007-06-10 20:31:24 -04:00
|
|
|
{
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
struct sk_buff *skb;
|
2007-06-10 20:31:24 -04:00
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
if ((skb = dev->gso_skb))
|
2007-06-10 20:31:24 -04:00
|
|
|
dev->gso_skb = NULL;
|
|
|
|
else
|
|
|
|
skb = q->dequeue(q);
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
static inline int handle_dev_cpu_collision(struct sk_buff *skb,
|
|
|
|
struct net_device *dev,
|
|
|
|
struct Qdisc *q)
|
2007-06-10 20:31:24 -04:00
|
|
|
{
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
int ret;
|
2007-06-10 20:31:24 -04:00
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
if (unlikely(dev->xmit_lock_owner == smp_processor_id())) {
|
|
|
|
/*
|
|
|
|
* Same CPU holding the lock. It may be a transient
|
|
|
|
* configuration error, when hard_start_xmit() recurses. We
|
|
|
|
* detect it by checking xmit owner and drop the packet when
|
|
|
|
* deadloop is detected. Return OK to try the next skb.
|
|
|
|
*/
|
2007-06-10 20:31:24 -04:00
|
|
|
kfree_skb(skb);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
if (net_ratelimit())
|
|
|
|
printk(KERN_WARNING "Dead loop on netdevice %s, "
|
|
|
|
"fix it urgently!\n", dev->name);
|
|
|
|
ret = qdisc_qlen(q);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Another cpu is holding lock, requeue & delay xmits for
|
|
|
|
* some time.
|
|
|
|
*/
|
|
|
|
__get_cpu_var(netdev_rx_stat).cpu_collision++;
|
|
|
|
ret = dev_requeue_skb(skb, dev, q);
|
2007-06-10 20:31:24 -04:00
|
|
|
}
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
return ret;
|
2007-06-10 20:31:24 -04:00
|
|
|
}
|
|
|
|
|
2007-02-09 09:25:16 -05:00
|
|
|
/*
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
* NOTE: Called under dev->queue_lock with locally disabled BH.
|
|
|
|
*
|
|
|
|
* __LINK_STATE_QDISC_RUNNING guarantees only one CPU can process this
|
|
|
|
* device at a time. dev->queue_lock serializes queue accesses for
|
|
|
|
* this device AND dev->qdisc pointer itself.
|
|
|
|
*
|
|
|
|
* netif_tx_lock serializes accesses to device driver.
|
|
|
|
*
|
|
|
|
* dev->queue_lock and netif_tx_lock are mutually exclusive,
|
|
|
|
* if one is grabbed, another must be free.
|
|
|
|
*
|
|
|
|
* Note, that this procedure can be called by a watchdog timer
|
|
|
|
*
|
|
|
|
* Returns to the caller:
|
|
|
|
* 0 - queue is empty or throttled.
|
|
|
|
* >0 - queue is not empty.
|
|
|
|
*
|
|
|
|
*/
|
2006-06-20 02:57:59 -04:00
|
|
|
static inline int qdisc_restart(struct net_device *dev)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
struct Qdisc *q = dev->qdisc;
|
|
|
|
struct sk_buff *skb;
|
2007-11-13 23:40:55 -05:00
|
|
|
int ret = NETDEV_TX_BUSY;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
/* Dequeue packet */
|
|
|
|
if (unlikely((skb = dev_dequeue_skb(dev, q)) == NULL))
|
2007-06-10 20:31:24 -04:00
|
|
|
return 0;
|
2006-06-22 05:57:17 -04:00
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
|
|
|
|
/* And release queue */
|
2007-06-10 20:31:24 -04:00
|
|
|
spin_unlock(&dev->queue_lock);
|
|
|
|
|
2007-09-25 22:27:13 -04:00
|
|
|
HARD_TX_LOCK(dev, smp_processor_id());
|
2007-11-13 23:40:55 -05:00
|
|
|
if (!netif_subqueue_stopped(dev, skb))
|
|
|
|
ret = dev_hard_start_xmit(skb, dev);
|
2007-09-25 22:27:13 -04:00
|
|
|
HARD_TX_UNLOCK(dev);
|
2007-06-10 20:31:24 -04:00
|
|
|
|
|
|
|
spin_lock(&dev->queue_lock);
|
|
|
|
q = dev->qdisc;
|
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
switch (ret) {
|
|
|
|
case NETDEV_TX_OK:
|
|
|
|
/* Driver sent out skb successfully */
|
|
|
|
ret = qdisc_qlen(q);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case NETDEV_TX_LOCKED:
|
|
|
|
/* Driver try lock failed */
|
|
|
|
ret = handle_dev_cpu_collision(skb, dev, q);
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
/* Driver returned NETDEV_TX_BUSY - requeue skb */
|
|
|
|
if (unlikely (ret != NETDEV_TX_BUSY && net_ratelimit()))
|
|
|
|
printk(KERN_WARNING "BUG %s code %d qlen %d\n",
|
|
|
|
dev->name, ret, q->q.qlen);
|
|
|
|
|
|
|
|
ret = dev_requeue_skb(skb, dev, q);
|
|
|
|
break;
|
|
|
|
}
|
2007-06-10 20:31:24 -04:00
|
|
|
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-24 22:56:09 -04:00
|
|
|
return ret;
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
2006-06-20 02:57:59 -04:00
|
|
|
void __qdisc_run(struct net_device *dev)
|
|
|
|
{
|
2007-05-10 07:55:14 -04:00
|
|
|
do {
|
|
|
|
if (!qdisc_restart(dev))
|
|
|
|
break;
|
|
|
|
} while (!netif_queue_stopped(dev));
|
2006-06-20 02:57:59 -04:00
|
|
|
|
|
|
|
clear_bit(__LINK_STATE_QDISC_RUNNING, &dev->state);
|
|
|
|
}
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
static void dev_watchdog(unsigned long arg)
|
|
|
|
{
|
|
|
|
struct net_device *dev = (struct net_device *)arg;
|
|
|
|
|
2006-06-09 15:20:56 -04:00
|
|
|
netif_tx_lock(dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
if (dev->qdisc != &noop_qdisc) {
|
|
|
|
if (netif_device_present(dev) &&
|
|
|
|
netif_running(dev) &&
|
|
|
|
netif_carrier_ok(dev)) {
|
|
|
|
if (netif_queue_stopped(dev) &&
|
2006-05-16 18:02:12 -04:00
|
|
|
time_after(jiffies, dev->trans_start + dev->watchdog_timeo)) {
|
|
|
|
|
|
|
|
printk(KERN_INFO "NETDEV WATCHDOG: %s: transmit timed out\n",
|
|
|
|
dev->name);
|
2005-04-16 18:20:36 -04:00
|
|
|
dev->tx_timeout(dev);
|
|
|
|
}
|
2007-02-05 20:59:51 -05:00
|
|
|
if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo)))
|
2005-04-16 18:20:36 -04:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
2006-06-09 15:20:56 -04:00
|
|
|
netif_tx_unlock(dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
dev_put(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __netdev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
|
|
|
if (dev->tx_timeout) {
|
|
|
|
if (dev->watchdog_timeo <= 0)
|
|
|
|
dev->watchdog_timeo = 5*HZ;
|
2007-06-01 00:28:44 -04:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
round_jiffies(jiffies + dev->watchdog_timeo)))
|
2005-04-16 18:20:36 -04:00
|
|
|
dev_hold(dev);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_up(struct net_device *dev)
|
|
|
|
{
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void dev_watchdog_down(struct net_device *dev)
|
|
|
|
{
|
2006-06-09 15:20:56 -04:00
|
|
|
netif_tx_lock_bh(dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
if (del_timer(&dev->watchdog_timer))
|
2006-03-21 01:32:28 -05:00
|
|
|
dev_put(dev);
|
2006-06-09 15:20:56 -04:00
|
|
|
netif_tx_unlock_bh(dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 19:41:36 -04:00
|
|
|
/**
|
|
|
|
* netif_carrier_on - set carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Device has detected that carrier.
|
|
|
|
*/
|
2005-08-11 18:32:53 -04:00
|
|
|
void netif_carrier_on(struct net_device *dev)
|
|
|
|
{
|
2007-10-18 02:26:43 -04:00
|
|
|
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
2005-08-11 18:32:53 -04:00
|
|
|
linkwatch_fire_event(dev);
|
2007-10-18 02:26:43 -04:00
|
|
|
if (netif_running(dev))
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
}
|
2005-08-11 18:32:53 -04:00
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(netif_carrier_on);
|
2005-08-11 18:32:53 -04:00
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 19:41:36 -04:00
|
|
|
/**
|
|
|
|
* netif_carrier_off - clear carrier
|
|
|
|
* @dev: network device
|
|
|
|
*
|
|
|
|
* Device has detected loss of carrier.
|
|
|
|
*/
|
2005-08-11 18:32:53 -04:00
|
|
|
void netif_carrier_off(struct net_device *dev)
|
|
|
|
{
|
|
|
|
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state))
|
|
|
|
linkwatch_fire_event(dev);
|
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(netif_carrier_off);
|
2005-08-11 18:32:53 -04:00
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
|
|
|
|
under all circumstances. It is difficult to invent anything faster or
|
|
|
|
cheaper.
|
|
|
|
*/
|
|
|
|
|
2005-06-19 01:59:08 -04:00
|
|
|
static int noop_enqueue(struct sk_buff *skb, struct Qdisc * qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
kfree_skb(skb);
|
|
|
|
return NET_XMIT_CN;
|
|
|
|
}
|
|
|
|
|
2005-06-19 01:59:08 -04:00
|
|
|
static struct sk_buff *noop_dequeue(struct Qdisc * qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2005-06-19 01:59:08 -04:00
|
|
|
static int noop_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
if (net_ratelimit())
|
2005-06-19 01:59:08 -04:00
|
|
|
printk(KERN_DEBUG "%s deferred output. It is buggy.\n",
|
|
|
|
skb->dev->name);
|
2005-04-16 18:20:36 -04:00
|
|
|
kfree_skb(skb);
|
|
|
|
return NET_XMIT_CN;
|
|
|
|
}
|
|
|
|
|
2007-11-14 04:44:41 -05:00
|
|
|
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
|
2005-04-16 18:20:36 -04:00
|
|
|
.id = "noop",
|
|
|
|
.priv_size = 0,
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.requeue = noop_requeue,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
|
|
|
struct Qdisc noop_qdisc = {
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
2007-02-09 09:25:16 -05:00
|
|
|
.ops = &noop_qdisc_ops,
|
2005-04-16 18:20:36 -04:00
|
|
|
.list = LIST_HEAD_INIT(noop_qdisc.list),
|
|
|
|
};
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(noop_qdisc);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2007-11-14 04:44:41 -05:00
|
|
|
static struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
|
2005-04-16 18:20:36 -04:00
|
|
|
.id = "noqueue",
|
|
|
|
.priv_size = 0,
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.requeue = noop_requeue,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
|
|
|
static struct Qdisc noqueue_qdisc = {
|
|
|
|
.enqueue = NULL,
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
|
|
|
.ops = &noqueue_qdisc_ops,
|
|
|
|
.list = LIST_HEAD_INIT(noqueue_qdisc.list),
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
static const u8 prio2band[TC_PRIO_MAX+1] =
|
|
|
|
{ 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1 };
|
|
|
|
|
|
|
|
/* 3-band FIFO queue: old style, but should be a bit faster than
|
|
|
|
generic prio+fifo combination.
|
|
|
|
*/
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
#define PFIFO_FAST_BANDS 3
|
|
|
|
|
2005-06-19 01:58:35 -04:00
|
|
|
static inline struct sk_buff_head *prio2list(struct sk_buff *skb,
|
|
|
|
struct Qdisc *qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
struct sk_buff_head *list = qdisc_priv(qdisc);
|
2005-06-19 01:58:35 -04:00
|
|
|
return list + prio2band[skb->priority & TC_PRIO_MAX];
|
|
|
|
}
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc* qdisc)
|
2005-06-19 01:58:35 -04:00
|
|
|
{
|
|
|
|
struct sk_buff_head *list = prio2list(skb, qdisc);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2005-06-19 01:58:15 -04:00
|
|
|
if (skb_queue_len(list) < qdisc->dev->tx_queue_len) {
|
2005-04-16 18:20:36 -04:00
|
|
|
qdisc->q.qlen++;
|
2005-06-19 01:58:15 -04:00
|
|
|
return __qdisc_enqueue_tail(skb, qdisc, list);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
2005-06-19 01:58:15 -04:00
|
|
|
|
|
|
|
return qdisc_drop(skb, qdisc);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc* qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
int prio;
|
|
|
|
struct sk_buff_head *list = qdisc_priv(qdisc);
|
|
|
|
|
2005-07-18 16:30:53 -04:00
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
if (!skb_queue_empty(list + prio)) {
|
2005-04-16 18:20:36 -04:00
|
|
|
qdisc->q.qlen--;
|
2005-07-18 16:30:53 -04:00
|
|
|
return __qdisc_dequeue_head(qdisc, list + prio);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
}
|
2005-06-19 01:58:53 -04:00
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
static int pfifo_fast_requeue(struct sk_buff *skb, struct Qdisc* qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
qdisc->q.qlen++;
|
2005-06-19 01:58:35 -04:00
|
|
|
return __qdisc_requeue(skb, qdisc, prio2list(skb, qdisc));
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
static void pfifo_fast_reset(struct Qdisc* qdisc)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
int prio;
|
|
|
|
struct sk_buff_head *list = qdisc_priv(qdisc);
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
2005-06-19 01:58:15 -04:00
|
|
|
__qdisc_reset_queue(qdisc, list + prio);
|
|
|
|
|
|
|
|
qdisc->qstats.backlog = 0;
|
2005-04-16 18:20:36 -04:00
|
|
|
qdisc->q.qlen = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
|
|
|
|
{
|
2005-06-19 01:58:53 -04:00
|
|
|
struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
memcpy(&opt.priomap, prio2band, TC_PRIO_MAX+1);
|
2008-01-23 01:11:17 -05:00
|
|
|
NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
|
2005-04-16 18:20:36 -04:00
|
|
|
return skb->len;
|
|
|
|
|
2008-01-23 01:11:17 -05:00
|
|
|
nla_put_failure:
|
2005-04-16 18:20:36 -04:00
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2008-01-23 01:11:17 -05:00
|
|
|
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
2005-06-19 01:58:53 -04:00
|
|
|
int prio;
|
2005-04-16 18:20:36 -04:00
|
|
|
struct sk_buff_head *list = qdisc_priv(qdisc);
|
|
|
|
|
2005-06-19 01:58:53 -04:00
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++)
|
|
|
|
skb_queue_head_init(list + prio);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-11-14 04:44:41 -05:00
|
|
|
static struct Qdisc_ops pfifo_fast_ops __read_mostly = {
|
2005-04-16 18:20:36 -04:00
|
|
|
.id = "pfifo_fast",
|
2005-06-19 01:58:53 -04:00
|
|
|
.priv_size = PFIFO_FAST_BANDS * sizeof(struct sk_buff_head),
|
2005-04-16 18:20:36 -04:00
|
|
|
.enqueue = pfifo_fast_enqueue,
|
|
|
|
.dequeue = pfifo_fast_dequeue,
|
|
|
|
.requeue = pfifo_fast_requeue,
|
|
|
|
.init = pfifo_fast_init,
|
|
|
|
.reset = pfifo_fast_reset,
|
|
|
|
.dump = pfifo_fast_dump,
|
|
|
|
.owner = THIS_MODULE,
|
|
|
|
};
|
|
|
|
|
2005-07-05 17:15:09 -04:00
|
|
|
struct Qdisc *qdisc_alloc(struct net_device *dev, struct Qdisc_ops *ops)
|
2005-04-16 18:20:36 -04:00
|
|
|
{
|
|
|
|
void *p;
|
|
|
|
struct Qdisc *sch;
|
2005-07-05 17:15:09 -04:00
|
|
|
unsigned int size;
|
|
|
|
int err = -ENOBUFS;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
/* ensure that the Qdisc and the private data are 32-byte aligned */
|
2005-07-05 17:15:09 -04:00
|
|
|
size = QDISC_ALIGN(sizeof(*sch));
|
|
|
|
size += ops->priv_size + (QDISC_ALIGNTO - 1);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2006-07-21 17:51:30 -04:00
|
|
|
p = kzalloc(size, GFP_KERNEL);
|
2005-04-16 18:20:36 -04:00
|
|
|
if (!p)
|
2005-07-05 17:15:09 -04:00
|
|
|
goto errout;
|
|
|
|
sch = (struct Qdisc *) QDISC_ALIGN((unsigned long) p);
|
|
|
|
sch->padded = (char *) sch - (char *) p;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
INIT_LIST_HEAD(&sch->list);
|
|
|
|
skb_queue_head_init(&sch->q);
|
|
|
|
sch->ops = ops;
|
|
|
|
sch->enqueue = ops->enqueue;
|
|
|
|
sch->dequeue = ops->dequeue;
|
|
|
|
sch->dev = dev;
|
|
|
|
dev_hold(dev);
|
|
|
|
atomic_set(&sch->refcnt, 1);
|
2005-07-05 17:15:09 -04:00
|
|
|
|
|
|
|
return sch;
|
|
|
|
errout:
|
|
|
|
return ERR_PTR(-err);
|
|
|
|
}
|
|
|
|
|
2006-11-29 20:35:18 -05:00
|
|
|
struct Qdisc * qdisc_create_dflt(struct net_device *dev, struct Qdisc_ops *ops,
|
|
|
|
unsigned int parentid)
|
2005-07-05 17:15:09 -04:00
|
|
|
{
|
|
|
|
struct Qdisc *sch;
|
2007-02-09 09:25:16 -05:00
|
|
|
|
2005-07-05 17:15:09 -04:00
|
|
|
sch = qdisc_alloc(dev, ops);
|
|
|
|
if (IS_ERR(sch))
|
|
|
|
goto errout;
|
2007-04-16 20:07:08 -04:00
|
|
|
sch->stats_lock = &dev->queue_lock;
|
2006-11-29 20:35:18 -05:00
|
|
|
sch->parent = parentid;
|
2005-07-05 17:15:09 -04:00
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
if (!ops->init || ops->init(sch, NULL) == 0)
|
|
|
|
return sch;
|
|
|
|
|
2005-08-23 13:12:44 -04:00
|
|
|
qdisc_destroy(sch);
|
2005-07-05 17:15:09 -04:00
|
|
|
errout:
|
2005-04-16 18:20:36 -04:00
|
|
|
return NULL;
|
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(qdisc_create_dflt);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
/* Under dev->queue_lock and BH! */
|
|
|
|
|
|
|
|
void qdisc_reset(struct Qdisc *qdisc)
|
|
|
|
{
|
2007-11-14 04:44:41 -05:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(qdisc_reset);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2007-02-09 09:25:16 -05:00
|
|
|
/* this is the rcu callback function to clean up a qdisc when there
|
2005-04-16 18:20:36 -04:00
|
|
|
* are no further references to it */
|
|
|
|
|
|
|
|
static void __qdisc_destroy(struct rcu_head *head)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc = container_of(head, struct Qdisc, q_rcu);
|
|
|
|
kfree((char *) qdisc - qdisc->padded);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Under dev->queue_lock and BH! */
|
|
|
|
|
|
|
|
void qdisc_destroy(struct Qdisc *qdisc)
|
|
|
|
{
|
2007-11-14 04:44:41 -05:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
[NET_SCHED]: Fix fallout from dev->qdisc RCU change
The move of qdisc destruction to a rcu callback broke locking in the
entire qdisc layer by invalidating previously valid assumptions about
the context in which changes to the qdisc tree occur.
The two assumptions were:
- since changes only happen in process context, read_lock doesn't need
bottem half protection. Now invalid since destruction of inner qdiscs,
classifiers, actions and estimators happens in the RCU callback unless
they're manually deleted, resulting in dead-locks when read_lock in
process context is interrupted by write_lock_bh in bottem half context.
- since changes only happen under the RTNL, no additional locking is
necessary for data not used during packet processing (f.e. u32_list).
Again, since destruction now happens in the RCU callback, this assumption
is not valid anymore, causing races while using this data, which can
result in corruption or use-after-free.
Instead of "fixing" this by disabling bottem halfs everywhere and adding
new locks/refcounting, this patch makes these assumptions valid again by
moving destruction back to process context. Since only the dev->qdisc
pointer is protected by RCU, but ->enqueue and the qdisc tree are still
protected by dev->qdisc_lock, destruction of the tree can be performed
immediately and only the final free needs to happen in the rcu callback
to make sure dev_queue_xmit doesn't access already freed memory.
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-09-27 19:45:45 -04:00
|
|
|
!atomic_dec_and_test(&qdisc->refcnt))
|
2005-04-16 18:20:36 -04:00
|
|
|
return;
|
|
|
|
|
[NET_SCHED]: Fix fallout from dev->qdisc RCU change
The move of qdisc destruction to a rcu callback broke locking in the
entire qdisc layer by invalidating previously valid assumptions about
the context in which changes to the qdisc tree occur.
The two assumptions were:
- since changes only happen in process context, read_lock doesn't need
bottem half protection. Now invalid since destruction of inner qdiscs,
classifiers, actions and estimators happens in the RCU callback unless
they're manually deleted, resulting in dead-locks when read_lock in
process context is interrupted by write_lock_bh in bottem half context.
- since changes only happen under the RTNL, no additional locking is
necessary for data not used during packet processing (f.e. u32_list).
Again, since destruction now happens in the RCU callback, this assumption
is not valid anymore, causing races while using this data, which can
result in corruption or use-after-free.
Instead of "fixing" this by disabling bottem halfs everywhere and adding
new locks/refcounting, this patch makes these assumptions valid again by
moving destruction back to process context. Since only the dev->qdisc
pointer is protected by RCU, but ->enqueue and the qdisc tree are still
protected by dev->qdisc_lock, destruction of the tree can be performed
immediately and only the final free needs to happen in the rcu callback
to make sure dev_queue_xmit doesn't access already freed memory.
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-09-27 19:45:45 -04:00
|
|
|
list_del(&qdisc->list);
|
|
|
|
gen_kill_estimator(&qdisc->bstats, &qdisc->rate_est);
|
|
|
|
if (ops->reset)
|
|
|
|
ops->reset(qdisc);
|
|
|
|
if (ops->destroy)
|
|
|
|
ops->destroy(qdisc);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
[NET_SCHED]: Fix fallout from dev->qdisc RCU change
The move of qdisc destruction to a rcu callback broke locking in the
entire qdisc layer by invalidating previously valid assumptions about
the context in which changes to the qdisc tree occur.
The two assumptions were:
- since changes only happen in process context, read_lock doesn't need
bottem half protection. Now invalid since destruction of inner qdiscs,
classifiers, actions and estimators happens in the RCU callback unless
they're manually deleted, resulting in dead-locks when read_lock in
process context is interrupted by write_lock_bh in bottem half context.
- since changes only happen under the RTNL, no additional locking is
necessary for data not used during packet processing (f.e. u32_list).
Again, since destruction now happens in the RCU callback, this assumption
is not valid anymore, causing races while using this data, which can
result in corruption or use-after-free.
Instead of "fixing" this by disabling bottem halfs everywhere and adding
new locks/refcounting, this patch makes these assumptions valid again by
moving destruction back to process context. Since only the dev->qdisc
pointer is protected by RCU, but ->enqueue and the qdisc tree are still
protected by dev->qdisc_lock, destruction of the tree can be performed
immediately and only the final free needs to happen in the rcu callback
to make sure dev_queue_xmit doesn't access already freed memory.
Signed-off-by: Patrick McHardy <kaber@trash.net>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-09-27 19:45:45 -04:00
|
|
|
module_put(ops->owner);
|
|
|
|
dev_put(qdisc->dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
call_rcu(&qdisc->q_rcu, __qdisc_destroy);
|
|
|
|
}
|
2008-01-23 01:10:23 -05:00
|
|
|
EXPORT_SYMBOL(qdisc_destroy);
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
void dev_activate(struct net_device *dev)
|
|
|
|
{
|
|
|
|
/* No queueing discipline is attached to device;
|
|
|
|
create default one i.e. pfifo_fast for devices,
|
|
|
|
which need queueing and noqueue_qdisc for
|
|
|
|
virtual interfaces
|
|
|
|
*/
|
|
|
|
|
|
|
|
if (dev->qdisc_sleeping == &noop_qdisc) {
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
if (dev->tx_queue_len) {
|
2006-11-29 20:35:18 -05:00
|
|
|
qdisc = qdisc_create_dflt(dev, &pfifo_fast_ops,
|
|
|
|
TC_H_ROOT);
|
2005-04-16 18:20:36 -04:00
|
|
|
if (qdisc == NULL) {
|
|
|
|
printk(KERN_INFO "%s: activation failed\n", dev->name);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
list_add_tail(&qdisc->list, &dev->qdisc_list);
|
|
|
|
} else {
|
|
|
|
qdisc = &noqueue_qdisc;
|
|
|
|
}
|
|
|
|
dev->qdisc_sleeping = qdisc;
|
|
|
|
}
|
|
|
|
|
2005-05-03 19:18:52 -04:00
|
|
|
if (!netif_carrier_ok(dev))
|
|
|
|
/* Delay activation until next carrier-on event */
|
|
|
|
return;
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
spin_lock_bh(&dev->queue_lock);
|
|
|
|
rcu_assign_pointer(dev->qdisc, dev->qdisc_sleeping);
|
|
|
|
if (dev->qdisc != &noqueue_qdisc) {
|
|
|
|
dev->trans_start = jiffies;
|
|
|
|
dev_watchdog_up(dev);
|
|
|
|
}
|
|
|
|
spin_unlock_bh(&dev->queue_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
void dev_deactivate(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc;
|
2007-05-10 17:12:47 -04:00
|
|
|
struct sk_buff *skb;
|
2007-10-19 01:37:58 -04:00
|
|
|
int running;
|
2005-04-16 18:20:36 -04:00
|
|
|
|
|
|
|
spin_lock_bh(&dev->queue_lock);
|
|
|
|
qdisc = dev->qdisc;
|
|
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
|
|
|
|
qdisc_reset(qdisc);
|
|
|
|
|
2007-05-10 17:12:47 -04:00
|
|
|
skb = dev->gso_skb;
|
|
|
|
dev->gso_skb = NULL;
|
2005-04-16 18:20:36 -04:00
|
|
|
spin_unlock_bh(&dev->queue_lock);
|
|
|
|
|
2007-05-10 17:12:47 -04:00
|
|
|
kfree_skb(skb);
|
|
|
|
|
2005-04-16 18:20:36 -04:00
|
|
|
dev_watchdog_down(dev);
|
|
|
|
|
2007-10-19 01:37:58 -04:00
|
|
|
/* Wait for outstanding qdisc-less dev_queue_xmit calls. */
|
2006-06-22 05:28:18 -04:00
|
|
|
synchronize_rcu();
|
2005-04-16 18:20:36 -04:00
|
|
|
|
2006-06-22 05:28:18 -04:00
|
|
|
/* Wait for outstanding qdisc_run calls. */
|
2007-10-19 01:37:58 -04:00
|
|
|
do {
|
|
|
|
while (test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state))
|
|
|
|
yield();
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Double-check inside queue lock to ensure that all effects
|
|
|
|
* of the queue run are visible when we return.
|
|
|
|
*/
|
|
|
|
spin_lock_bh(&dev->queue_lock);
|
|
|
|
running = test_bit(__LINK_STATE_QDISC_RUNNING, &dev->state);
|
|
|
|
spin_unlock_bh(&dev->queue_lock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The running flag should never be set at this point because
|
|
|
|
* we've already set dev->qdisc to noop_qdisc *inside* the same
|
|
|
|
* pair of spin locks. That is, if any qdisc_run starts after
|
|
|
|
* our initial test it should see the noop_qdisc and then
|
|
|
|
* clear the RUNNING bit before dropping the queue lock. So
|
|
|
|
* if it is set here then we've found a bug.
|
|
|
|
*/
|
|
|
|
} while (WARN_ON_ONCE(running));
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
void dev_init_scheduler(struct net_device *dev)
|
|
|
|
{
|
|
|
|
qdisc_lock_tree(dev);
|
|
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
dev->qdisc_sleeping = &noop_qdisc;
|
|
|
|
INIT_LIST_HEAD(&dev->qdisc_list);
|
|
|
|
qdisc_unlock_tree(dev);
|
|
|
|
|
2008-01-24 00:20:07 -05:00
|
|
|
setup_timer(&dev->watchdog_timer, dev_watchdog, (unsigned long)dev);
|
2005-04-16 18:20:36 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
void dev_shutdown(struct net_device *dev)
|
|
|
|
{
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
qdisc_lock_tree(dev);
|
|
|
|
qdisc = dev->qdisc_sleeping;
|
|
|
|
dev->qdisc = &noop_qdisc;
|
|
|
|
dev->qdisc_sleeping = &noop_qdisc;
|
|
|
|
qdisc_destroy(qdisc);
|
|
|
|
#if defined(CONFIG_NET_SCH_INGRESS) || defined(CONFIG_NET_SCH_INGRESS_MODULE)
|
2007-02-09 09:25:16 -05:00
|
|
|
if ((qdisc = dev->qdisc_ingress) != NULL) {
|
2005-04-16 18:20:36 -04:00
|
|
|
dev->qdisc_ingress = NULL;
|
|
|
|
qdisc_destroy(qdisc);
|
2007-02-09 09:25:16 -05:00
|
|
|
}
|
2005-04-16 18:20:36 -04:00
|
|
|
#endif
|
|
|
|
BUG_TRAP(!timer_pending(&dev->watchdog_timer));
|
|
|
|
qdisc_unlock_tree(dev);
|
|
|
|
}
|