sch_htb.c 42.0 KB
Newer Older
S
Stephen Hemminger 已提交
1
/*
L
Linus Torvalds 已提交
2 3 4 5 6 7 8 9 10 11 12 13
 * net/sched/sch_htb.c	Hierarchical token bucket, feed tree version
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 * Authors:	Martin Devera, <devik@cdi.cz>
 *
 * Credits (in time order) for older HTB versions:
 *              Stef Coene <stef.coene@docum.org>
 *			HTB support at LARTC mailing list
14
 *		Ondrej Kraus, <krauso@barr.cz>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *			found missing INIT_QDISC(htb)
 *		Vladimir Smelhaus, Aamer Akhter, Bert Hubert
 *			helped a lot to locate nasty class stall bug
 *		Andi Kleen, Jamal Hadi, Bert Hubert
 *			code review and helpful comments on shaping
 *		Tomasz Wrona, <tw@eter.tym.pl>
 *			created test case so that I was able to fix nasty bug
 *		Wilfried Weissmann
 *			spotted bug in dequeue code and helped with fix
 *		Jiri Fojtasek
 *			fixed requeue routine
 *		and many others. thanks.
 */
#include <linux/module.h>
29
#include <linux/moduleparam.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35 36
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/list.h>
#include <linux/compiler.h>
37
#include <linux/rbtree.h>
38
#include <linux/workqueue.h>
39
#include <net/netlink.h>
L
Linus Torvalds 已提交
40 41 42 43 44 45
#include <net/pkt_sched.h>

/* HTB algorithm.
    Author: devik@cdi.cz
    ========================================================================
    HTB is like TBF with multiple classes. It is also similar to CBQ because
46
    it allows to assign priority to each class in hierarchy.
L
Linus Torvalds 已提交
47 48 49
    In fact it is another implementation of Floyd's formal sharing.

    Levels:
50
    Each class is assigned level. Leaf has ALWAYS level 0 and root
L
Linus Torvalds 已提交
51 52 53 54
    classes have level TC_HTB_MAXDEPTH-1. Interior nodes has level
    one less than their parent.
*/

55
static int htb_hysteresis __read_mostly = 0; /* whether to use mode hysteresis for speedup */
S
Stephen Hemminger 已提交
56
#define HTB_VER 0x30011		/* major must be matched with number suplied by TC as version */
L
Linus Torvalds 已提交
57 58 59 60 61

#if HTB_VER >> 16 != TC_HTB_PROTOVER
#error "Mismatched sch_htb.c and pkt_sch.h"
#endif

62 63 64 65
/* Module parameter and sysfs export */
module_param    (htb_hysteresis, int, 0640);
MODULE_PARM_DESC(htb_hysteresis, "Hysteresis mode, less CPU load, less accurate");

L
Linus Torvalds 已提交
66 67
/* used internaly to keep status of single class */
enum htb_cmode {
S
Stephen Hemminger 已提交
68 69 70
	HTB_CANT_SEND,		/* class can't send and can't borrow */
	HTB_MAY_BORROW,		/* class can't send but may borrow */
	HTB_CAN_SEND		/* class can send */
L
Linus Torvalds 已提交
71 72 73
};

/* interior & leaf nodes; props specific to leaves are marked L: */
S
Stephen Hemminger 已提交
74
struct htb_class {
75
	struct Qdisc_class_common common;
S
Stephen Hemminger 已提交
76 77 78 79 80 81
	/* general class parameters */
	struct gnet_stats_basic bstats;
	struct gnet_stats_queue qstats;
	struct gnet_stats_rate_est rate_est;
	struct tc_htb_xstats xstats;	/* our special stats */
	int refcnt;		/* usage count of this class */
L
Linus Torvalds 已提交
82

S
Stephen Hemminger 已提交
83 84
	/* topology */
	int level;		/* our level (see above) */
85
	unsigned int children;
S
Stephen Hemminger 已提交
86 87
	struct htb_class *parent;	/* parent class */

88 89 90
	int prio;		/* these two are used only by leaves... */
	int quantum;		/* but stored for parent-to-leaf return */

S
Stephen Hemminger 已提交
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
	union {
		struct htb_class_leaf {
			struct Qdisc *q;
			int deficit[TC_HTB_MAXDEPTH];
			struct list_head drop_list;
		} leaf;
		struct htb_class_inner {
			struct rb_root feed[TC_HTB_NUMPRIO];	/* feed trees */
			struct rb_node *ptr[TC_HTB_NUMPRIO];	/* current class ptr */
			/* When class changes from state 1->2 and disconnects from
			   parent's feed then we lost ptr value and start from the
			   first child again. Here we store classid of the
			   last valid ptr (used when ptr is NULL). */
			u32 last_ptr_id[TC_HTB_NUMPRIO];
		} inner;
	} un;
	struct rb_node node[TC_HTB_NUMPRIO];	/* node for self or feed tree */
	struct rb_node pq_node;	/* node for event queue */
109
	psched_time_t pq_key;
S
Stephen Hemminger 已提交
110 111 112 113 114 115 116 117 118 119 120 121 122 123 124

	int prio_activity;	/* for which prios are we active */
	enum htb_cmode cmode;	/* current mode of the class */

	/* class attached filters */
	struct tcf_proto *filter_list;
	int filter_cnt;

	/* token bucket parameters */
	struct qdisc_rate_table *rate;	/* rate table of the class itself */
	struct qdisc_rate_table *ceil;	/* ceiling rate (limits borrows too) */
	long buffer, cbuffer;	/* token bucket depth/rate */
	psched_tdiff_t mbuffer;	/* max wait time */
	long tokens, ctokens;	/* current number of tokens */
	psched_time_t t_c;	/* checkpoint time */
L
Linus Torvalds 已提交
125 126
};

S
Stephen Hemminger 已提交
127
struct htb_sched {
128
	struct Qdisc_class_hash clhash;
129
	struct list_head drops[TC_HTB_NUMPRIO];/* active leaves (for drops) */
S
Stephen Hemminger 已提交
130 131 132 133 134 135

	/* self list - roots of self generating tree */
	struct rb_root row[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
	int row_mask[TC_HTB_MAXDEPTH];
	struct rb_node *ptr[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
	u32 last_ptr_id[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
L
Linus Torvalds 已提交
136

S
Stephen Hemminger 已提交
137 138
	/* self wait list - roots of wait PQs per row */
	struct rb_root wait_pq[TC_HTB_MAXDEPTH];
L
Linus Torvalds 已提交
139

S
Stephen Hemminger 已提交
140
	/* time of nearest event per level (row) */
141
	psched_time_t near_ev_cache[TC_HTB_MAXDEPTH];
L
Linus Torvalds 已提交
142

S
Stephen Hemminger 已提交
143
	int defcls;		/* class where unclassified flows go to */
L
Linus Torvalds 已提交
144

S
Stephen Hemminger 已提交
145 146
	/* filters for qdisc itself */
	struct tcf_proto *filter_list;
L
Linus Torvalds 已提交
147

S
Stephen Hemminger 已提交
148 149
	int rate2quantum;	/* quant = rate / rate2quantum */
	psched_time_t now;	/* cached dequeue time */
150
	struct qdisc_watchdog watchdog;
L
Linus Torvalds 已提交
151

S
Stephen Hemminger 已提交
152 153 154 155 156
	/* non shaped skbs; let them go directly thru */
	struct sk_buff_head direct_queue;
	int direct_qlen;	/* max qlen of above */

	long direct_pkts;
157 158 159

#define HTB_WARN_TOOMANYEVENTS	0x1
	unsigned int warned;	/* only one warning */
160
	struct work_struct work;
L
Linus Torvalds 已提交
161 162 163
};

/* find class in global hash table using given handle */
S
Stephen Hemminger 已提交
164
static inline struct htb_class *htb_find(u32 handle, struct Qdisc *sch)
L
Linus Torvalds 已提交
165 166
{
	struct htb_sched *q = qdisc_priv(sch);
167
	struct Qdisc_class_common *clc;
168

169 170
	clc = qdisc_class_find(&q->clhash, handle);
	if (clc == NULL)
L
Linus Torvalds 已提交
171
		return NULL;
172
	return container_of(clc, struct htb_class, common);
L
Linus Torvalds 已提交
173 174 175 176 177 178 179 180 181 182
}

/**
 * htb_classify - classify a packet into class
 *
 * It returns NULL if the packet should be dropped or -1 if the packet
 * should be passed directly thru. In all other cases leaf class is returned.
 * We allow direct class selection by classid in priority. The we examine
 * filters in qdisc and in inner nodes (if higher filter points to the inner
 * node). If we end up with classid MAJOR:0 we enqueue the skb into special
183
 * internal fifo (direct). These packets then go directly thru. If we still
L
Linus Torvalds 已提交
184 185 186 187 188
 * have no valid leaf we try to use MAJOR:default leaf. It still unsuccessfull
 * then finish and return direct queue.
 */
#define HTB_DIRECT (struct htb_class*)-1

S
Stephen Hemminger 已提交
189 190
static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch,
				      int *qerr)
L
Linus Torvalds 已提交
191 192 193 194 195 196 197 198 199 200 201
{
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl;
	struct tcf_result res;
	struct tcf_proto *tcf;
	int result;

	/* allow to select class by setting skb->priority to valid classid;
	   note that nfmark can be used too by attaching filter fw with no
	   rules in it */
	if (skb->priority == sch->handle)
S
Stephen Hemminger 已提交
202 203
		return HTB_DIRECT;	/* X:0 (direct flow) selected */
	if ((cl = htb_find(skb->priority, sch)) != NULL && cl->level == 0)
L
Linus Torvalds 已提交
204 205
		return cl;

206
	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
L
Linus Torvalds 已提交
207 208 209 210 211
	tcf = q->filter_list;
	while (tcf && (result = tc_classify(skb, tcf, &res)) >= 0) {
#ifdef CONFIG_NET_CLS_ACT
		switch (result) {
		case TC_ACT_QUEUED:
S
Stephen Hemminger 已提交
212
		case TC_ACT_STOLEN:
213
			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
L
Linus Torvalds 已提交
214 215 216 217
		case TC_ACT_SHOT:
			return NULL;
		}
#endif
S
Stephen Hemminger 已提交
218
		if ((cl = (void *)res.class) == NULL) {
L
Linus Torvalds 已提交
219
			if (res.classid == sch->handle)
S
Stephen Hemminger 已提交
220 221 222
				return HTB_DIRECT;	/* X:0 (direct flow) */
			if ((cl = htb_find(res.classid, sch)) == NULL)
				break;	/* filter selected invalid classid */
L
Linus Torvalds 已提交
223 224
		}
		if (!cl->level)
S
Stephen Hemminger 已提交
225
			return cl;	/* we hit leaf; return it */
L
Linus Torvalds 已提交
226 227 228 229 230

		/* we have got inner class; apply inner filter chain */
		tcf = cl->filter_list;
	}
	/* classification failed; try to use default class */
S
Stephen Hemminger 已提交
231
	cl = htb_find(TC_H_MAKE(TC_H_MAJ(sch->handle), q->defcls), sch);
L
Linus Torvalds 已提交
232
	if (!cl || cl->level)
S
Stephen Hemminger 已提交
233
		return HTB_DIRECT;	/* bad default .. this is safe bet */
L
Linus Torvalds 已提交
234 235 236 237 238 239 240 241 242
	return cl;
}

/**
 * htb_add_to_id_tree - adds class to the round robin list
 *
 * Routine adds class to the list (actually tree) sorted by classid.
 * Make sure that class is not already on such list for given prio.
 */
S
Stephen Hemminger 已提交
243 244
static void htb_add_to_id_tree(struct rb_root *root,
			       struct htb_class *cl, int prio)
L
Linus Torvalds 已提交
245 246
{
	struct rb_node **p = &root->rb_node, *parent = NULL;
247

L
Linus Torvalds 已提交
248
	while (*p) {
S
Stephen Hemminger 已提交
249 250
		struct htb_class *c;
		parent = *p;
L
Linus Torvalds 已提交
251
		c = rb_entry(parent, struct htb_class, node[prio]);
252

253
		if (cl->common.classid > c->common.classid)
L
Linus Torvalds 已提交
254
			p = &parent->rb_right;
S
Stephen Hemminger 已提交
255
		else
L
Linus Torvalds 已提交
256 257 258 259 260 261 262 263 264 265 266 267 268
			p = &parent->rb_left;
	}
	rb_link_node(&cl->node[prio], parent, p);
	rb_insert_color(&cl->node[prio], root);
}

/**
 * htb_add_to_wait_tree - adds class to the event queue with delay
 *
 * The class is added to priority event queue to indicate that class will
 * change its mode in cl->pq_key microseconds. Make sure that class is not
 * already in the queue.
 */
S
Stephen Hemminger 已提交
269 270
static void htb_add_to_wait_tree(struct htb_sched *q,
				 struct htb_class *cl, long delay)
L
Linus Torvalds 已提交
271 272
{
	struct rb_node **p = &q->wait_pq[cl->level].rb_node, *parent = NULL;
273

274 275
	cl->pq_key = q->now + delay;
	if (cl->pq_key == q->now)
L
Linus Torvalds 已提交
276 277 278
		cl->pq_key++;

	/* update the nearest event cache */
279
	if (q->near_ev_cache[cl->level] > cl->pq_key)
L
Linus Torvalds 已提交
280
		q->near_ev_cache[cl->level] = cl->pq_key;
S
Stephen Hemminger 已提交
281

L
Linus Torvalds 已提交
282
	while (*p) {
S
Stephen Hemminger 已提交
283 284
		struct htb_class *c;
		parent = *p;
L
Linus Torvalds 已提交
285
		c = rb_entry(parent, struct htb_class, pq_node);
286
		if (cl->pq_key >= c->pq_key)
L
Linus Torvalds 已提交
287
			p = &parent->rb_right;
S
Stephen Hemminger 已提交
288
		else
L
Linus Torvalds 已提交
289 290 291 292 293 294 295 296 297 298 299 300
			p = &parent->rb_left;
	}
	rb_link_node(&cl->pq_node, parent, p);
	rb_insert_color(&cl->pq_node, &q->wait_pq[cl->level]);
}

/**
 * htb_next_rb_node - finds next node in binary tree
 *
 * When we are past last key we return NULL.
 * Average complexity is 2 steps per call.
 */
S
Stephen Hemminger 已提交
301
static inline void htb_next_rb_node(struct rb_node **n)
L
Linus Torvalds 已提交
302 303 304 305 306 307 308 309 310 311
{
	*n = rb_next(*n);
}

/**
 * htb_add_class_to_row - add class to its row
 *
 * The class is added to row at priorities marked in mask.
 * It does nothing if mask == 0.
 */
S
Stephen Hemminger 已提交
312 313
static inline void htb_add_class_to_row(struct htb_sched *q,
					struct htb_class *cl, int mask)
L
Linus Torvalds 已提交
314 315 316 317 318
{
	q->row_mask[cl->level] |= mask;
	while (mask) {
		int prio = ffz(~mask);
		mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
319
		htb_add_to_id_tree(q->row[cl->level] + prio, cl, prio);
L
Linus Torvalds 已提交
320 321 322
	}
}

S
Stephen Hemminger 已提交
323 324 325
/* If this triggers, it is a bug in this code, but it need not be fatal */
static void htb_safe_rb_erase(struct rb_node *rb, struct rb_root *root)
{
326
	if (RB_EMPTY_NODE(rb)) {
S
Stephen Hemminger 已提交
327 328 329 330 331 332 333 334
		WARN_ON(1);
	} else {
		rb_erase(rb, root);
		RB_CLEAR_NODE(rb);
	}
}


L
Linus Torvalds 已提交
335 336 337 338 339 340
/**
 * htb_remove_class_from_row - removes class from its row
 *
 * The class is removed from row at priorities marked in mask.
 * It does nothing if mask == 0.
 */
S
Stephen Hemminger 已提交
341 342
static inline void htb_remove_class_from_row(struct htb_sched *q,
						 struct htb_class *cl, int mask)
L
Linus Torvalds 已提交
343 344
{
	int m = 0;
345

L
Linus Torvalds 已提交
346 347
	while (mask) {
		int prio = ffz(~mask);
S
Stephen Hemminger 已提交
348

L
Linus Torvalds 已提交
349
		mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
350 351
		if (q->ptr[cl->level][prio] == cl->node + prio)
			htb_next_rb_node(q->ptr[cl->level] + prio);
S
Stephen Hemminger 已提交
352 353

		htb_safe_rb_erase(cl->node + prio, q->row[cl->level] + prio);
S
Stephen Hemminger 已提交
354
		if (!q->row[cl->level][prio].rb_node)
L
Linus Torvalds 已提交
355 356 357 358 359 360 361 362 363
			m |= 1 << prio;
	}
	q->row_mask[cl->level] &= ~m;
}

/**
 * htb_activate_prios - creates active classe's feed chain
 *
 * The class is connected to ancestors and/or appropriate rows
364
 * for priorities it is participating on. cl->cmode must be new
L
Linus Torvalds 已提交
365 366
 * (activated) mode. It does nothing if cl->prio_activity == 0.
 */
S
Stephen Hemminger 已提交
367
static void htb_activate_prios(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
368 369
{
	struct htb_class *p = cl->parent;
S
Stephen Hemminger 已提交
370
	long m, mask = cl->prio_activity;
L
Linus Torvalds 已提交
371 372

	while (cl->cmode == HTB_MAY_BORROW && p && mask) {
S
Stephen Hemminger 已提交
373 374
		m = mask;
		while (m) {
L
Linus Torvalds 已提交
375 376
			int prio = ffz(~m);
			m &= ~(1 << prio);
S
Stephen Hemminger 已提交
377

L
Linus Torvalds 已提交
378 379 380 381
			if (p->un.inner.feed[prio].rb_node)
				/* parent already has its feed in use so that
				   reset bit in mask as parent is already ok */
				mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
382 383

			htb_add_to_id_tree(p->un.inner.feed + prio, cl, prio);
L
Linus Torvalds 已提交
384 385
		}
		p->prio_activity |= mask;
S
Stephen Hemminger 已提交
386 387
		cl = p;
		p = cl->parent;
388

L
Linus Torvalds 已提交
389 390
	}
	if (cl->cmode == HTB_CAN_SEND && mask)
S
Stephen Hemminger 已提交
391
		htb_add_class_to_row(q, cl, mask);
L
Linus Torvalds 已提交
392 393 394 395 396
}

/**
 * htb_deactivate_prios - remove class from feed chain
 *
397
 * cl->cmode must represent old mode (before deactivation). It does
L
Linus Torvalds 已提交
398 399 400 401 402 403
 * nothing if cl->prio_activity == 0. Class is removed from all feed
 * chains and rows.
 */
static void htb_deactivate_prios(struct htb_sched *q, struct htb_class *cl)
{
	struct htb_class *p = cl->parent;
S
Stephen Hemminger 已提交
404
	long m, mask = cl->prio_activity;
L
Linus Torvalds 已提交
405 406

	while (cl->cmode == HTB_MAY_BORROW && p && mask) {
S
Stephen Hemminger 已提交
407 408
		m = mask;
		mask = 0;
L
Linus Torvalds 已提交
409 410 411
		while (m) {
			int prio = ffz(~m);
			m &= ~(1 << prio);
S
Stephen Hemminger 已提交
412 413

			if (p->un.inner.ptr[prio] == cl->node + prio) {
L
Linus Torvalds 已提交
414 415 416
				/* we are removing child which is pointed to from
				   parent feed - forget the pointer but remember
				   classid */
417
				p->un.inner.last_ptr_id[prio] = cl->common.classid;
L
Linus Torvalds 已提交
418 419
				p->un.inner.ptr[prio] = NULL;
			}
S
Stephen Hemminger 已提交
420

S
Stephen Hemminger 已提交
421
			htb_safe_rb_erase(cl->node + prio, p->un.inner.feed + prio);
S
Stephen Hemminger 已提交
422 423

			if (!p->un.inner.feed[prio].rb_node)
L
Linus Torvalds 已提交
424 425
				mask |= 1 << prio;
		}
426

L
Linus Torvalds 已提交
427
		p->prio_activity &= ~mask;
S
Stephen Hemminger 已提交
428 429
		cl = p;
		p = cl->parent;
430

L
Linus Torvalds 已提交
431
	}
S
Stephen Hemminger 已提交
432 433
	if (cl->cmode == HTB_CAN_SEND && mask)
		htb_remove_class_from_row(q, cl, mask);
L
Linus Torvalds 已提交
434 435
}

436 437
static inline long htb_lowater(const struct htb_class *cl)
{
438 439 440 441
	if (htb_hysteresis)
		return cl->cmode != HTB_CANT_SEND ? -cl->cbuffer : 0;
	else
		return 0;
442 443 444
}
static inline long htb_hiwater(const struct htb_class *cl)
{
445 446 447 448
	if (htb_hysteresis)
		return cl->cmode == HTB_CAN_SEND ? -cl->buffer : 0;
	else
		return 0;
449
}
450

451

L
Linus Torvalds 已提交
452 453 454 455 456
/**
 * htb_class_mode - computes and returns current class mode
 *
 * It computes cl's mode at time cl->t_c+diff and returns it. If mode
 * is not HTB_CAN_SEND then cl->pq_key is updated to time difference
457
 * from now to time when cl will change its state.
L
Linus Torvalds 已提交
458
 * Also it is worth to note that class mode doesn't change simply
459
 * at cl->{c,}tokens == 0 but there can rather be hysteresis of
L
Linus Torvalds 已提交
460 461 462
 * 0 .. -cl->{c,}buffer range. It is meant to limit number of
 * mode transitions per time unit. The speed gain is about 1/6.
 */
S
Stephen Hemminger 已提交
463 464
static inline enum htb_cmode
htb_class_mode(struct htb_class *cl, long *diff)
L
Linus Torvalds 已提交
465
{
S
Stephen Hemminger 已提交
466
	long toks;
L
Linus Torvalds 已提交
467

S
Stephen Hemminger 已提交
468 469 470 471
	if ((toks = (cl->ctokens + *diff)) < htb_lowater(cl)) {
		*diff = -toks;
		return HTB_CANT_SEND;
	}
472

S
Stephen Hemminger 已提交
473 474
	if ((toks = (cl->tokens + *diff)) >= htb_hiwater(cl))
		return HTB_CAN_SEND;
L
Linus Torvalds 已提交
475

S
Stephen Hemminger 已提交
476 477
	*diff = -toks;
	return HTB_MAY_BORROW;
L
Linus Torvalds 已提交
478 479 480 481 482 483 484 485 486 487 488
}

/**
 * htb_change_class_mode - changes classe's mode
 *
 * This should be the only way how to change classe's mode under normal
 * cirsumstances. Routine will update feed lists linkage, change mode
 * and add class to the wait event queue if appropriate. New mode should
 * be different from old one and cl->pq_key has to be valid if changing
 * to mode other than HTB_CAN_SEND (see htb_add_to_wait_tree).
 */
S
Stephen Hemminger 已提交
489
static void
L
Linus Torvalds 已提交
490
htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, long *diff)
S
Stephen Hemminger 已提交
491 492
{
	enum htb_cmode new_mode = htb_class_mode(cl, diff);
L
Linus Torvalds 已提交
493 494

	if (new_mode == cl->cmode)
S
Stephen Hemminger 已提交
495 496 497 498 499
		return;

	if (cl->prio_activity) {	/* not necessary: speed optimization */
		if (cl->cmode != HTB_CANT_SEND)
			htb_deactivate_prios(q, cl);
L
Linus Torvalds 已提交
500
		cl->cmode = new_mode;
S
Stephen Hemminger 已提交
501 502 503
		if (new_mode != HTB_CANT_SEND)
			htb_activate_prios(q, cl);
	} else
L
Linus Torvalds 已提交
504 505 506 507
		cl->cmode = new_mode;
}

/**
508
 * htb_activate - inserts leaf cl into appropriate active feeds
L
Linus Torvalds 已提交
509 510 511 512 513
 *
 * Routine learns (new) priority of leaf and activates feed chain
 * for the prio. It can be called on already active leaf safely.
 * It also adds leaf into droplist.
 */
S
Stephen Hemminger 已提交
514
static inline void htb_activate(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
515
{
516
	WARN_ON(cl->level || !cl->un.leaf.q || !cl->un.leaf.q->q.qlen);
517

L
Linus Torvalds 已提交
518
	if (!cl->prio_activity) {
519
		cl->prio_activity = 1 << cl->prio;
S
Stephen Hemminger 已提交
520 521
		htb_activate_prios(q, cl);
		list_add_tail(&cl->un.leaf.drop_list,
522
			      q->drops + cl->prio);
L
Linus Torvalds 已提交
523 524 525 526
	}
}

/**
527
 * htb_deactivate - remove leaf cl from active feeds
L
Linus Torvalds 已提交
528 529 530 531
 *
 * Make sure that leaf is active. In the other words it can't be called
 * with non-active leaf. It also removes class from the drop list.
 */
S
Stephen Hemminger 已提交
532
static inline void htb_deactivate(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
533
{
534
	WARN_ON(!cl->prio_activity);
535

S
Stephen Hemminger 已提交
536
	htb_deactivate_prios(q, cl);
L
Linus Torvalds 已提交
537 538 539 540 541 542
	cl->prio_activity = 0;
	list_del_init(&cl->un.leaf.drop_list);
}

static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
543
	int uninitialized_var(ret);
S
Stephen Hemminger 已提交
544 545 546 547 548 549 550 551 552 553 554 555 556
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl = htb_classify(skb, sch, &ret);

	if (cl == HTB_DIRECT) {
		/* enqueue to helper queue */
		if (q->direct_queue.qlen < q->direct_qlen) {
			__skb_queue_tail(&q->direct_queue, skb);
			q->direct_pkts++;
		} else {
			kfree_skb(skb);
			sch->qstats.drops++;
			return NET_XMIT_DROP;
		}
L
Linus Torvalds 已提交
557
#ifdef CONFIG_NET_CLS_ACT
S
Stephen Hemminger 已提交
558
	} else if (!cl) {
559
		if (ret & __NET_XMIT_BYPASS)
S
Stephen Hemminger 已提交
560 561 562
			sch->qstats.drops++;
		kfree_skb(skb);
		return ret;
L
Linus Torvalds 已提交
563
#endif
564 565 566 567 568
	} else if ((ret = qdisc_enqueue(skb, cl->un.leaf.q)) != NET_XMIT_SUCCESS) {
		if (net_xmit_drop_count(ret)) {
			sch->qstats.drops++;
			cl->qstats.drops++;
		}
569
		return ret;
S
Stephen Hemminger 已提交
570
	} else {
571 572
		cl->bstats.packets +=
			skb_is_gso(skb)?skb_shinfo(skb)->gso_segs:1;
573
		cl->bstats.bytes += qdisc_pkt_len(skb);
S
Stephen Hemminger 已提交
574 575 576 577
		htb_activate(q, cl);
	}

	sch->q.qlen++;
578
	sch->bstats.packets += skb_is_gso(skb)?skb_shinfo(skb)->gso_segs:1;
579
	sch->bstats.bytes += qdisc_pkt_len(skb);
S
Stephen Hemminger 已提交
580
	return NET_XMIT_SUCCESS;
L
Linus Torvalds 已提交
581 582
}

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
static inline void htb_accnt_tokens(struct htb_class *cl, int bytes, long diff)
{
	long toks = diff + cl->tokens;

	if (toks > cl->buffer)
		toks = cl->buffer;
	toks -= (long) qdisc_l2t(cl->rate, bytes);
	if (toks <= -cl->mbuffer)
		toks = 1 - cl->mbuffer;

	cl->tokens = toks;
}

static inline void htb_accnt_ctokens(struct htb_class *cl, int bytes, long diff)
{
	long toks = diff + cl->ctokens;

	if (toks > cl->cbuffer)
		toks = cl->cbuffer;
	toks -= (long) qdisc_l2t(cl->ceil, bytes);
	if (toks <= -cl->mbuffer)
		toks = 1 - cl->mbuffer;

	cl->ctokens = toks;
}

L
Linus Torvalds 已提交
609 610 611 612 613 614 615 616 617 618 619
/**
 * htb_charge_class - charges amount "bytes" to leaf and ancestors
 *
 * Routine assumes that packet "bytes" long was dequeued from leaf cl
 * borrowing from "level". It accounts bytes to ceil leaky bucket for
 * leaf and all ancestors and to rate bucket for ancestors at levels
 * "level" and higher. It also handles possible change of mode resulting
 * from the update. Note that mode can also increase here (MAY_BORROW to
 * CAN_SEND) because we can use more precise clock that event queue here.
 * In such case we remove class from event queue first.
 */
S
Stephen Hemminger 已提交
620
static void htb_charge_class(struct htb_sched *q, struct htb_class *cl,
621
			     int level, struct sk_buff *skb)
S
Stephen Hemminger 已提交
622
{
623
	int bytes = qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
624
	enum htb_cmode old_mode;
625
	long diff;
L
Linus Torvalds 已提交
626 627

	while (cl) {
628
		diff = psched_tdiff_bounded(q->now, cl->t_c, cl->mbuffer);
L
Linus Torvalds 已提交
629
		if (cl->level >= level) {
S
Stephen Hemminger 已提交
630 631
			if (cl->level == level)
				cl->xstats.lends++;
632
			htb_accnt_tokens(cl, bytes, diff);
L
Linus Torvalds 已提交
633 634
		} else {
			cl->xstats.borrows++;
S
Stephen Hemminger 已提交
635
			cl->tokens += diff;	/* we moved t_c; update tokens */
L
Linus Torvalds 已提交
636
		}
637
		htb_accnt_ctokens(cl, bytes, diff);
L
Linus Torvalds 已提交
638 639
		cl->t_c = q->now;

S
Stephen Hemminger 已提交
640 641 642
		old_mode = cl->cmode;
		diff = 0;
		htb_change_class_mode(q, cl, &diff);
L
Linus Torvalds 已提交
643 644
		if (old_mode != cl->cmode) {
			if (old_mode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
645
				htb_safe_rb_erase(&cl->pq_node, q->wait_pq + cl->level);
L
Linus Torvalds 已提交
646
			if (cl->cmode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
647
				htb_add_to_wait_tree(q, cl, diff);
L
Linus Torvalds 已提交
648 649 650 651 652
		}

		/* update byte stats except for leaves which are already updated */
		if (cl->level) {
			cl->bstats.bytes += bytes;
653 654
			cl->bstats.packets += skb_is_gso(skb)?
					skb_shinfo(skb)->gso_segs:1;
L
Linus Torvalds 已提交
655 656 657 658 659 660 661 662
		}
		cl = cl->parent;
	}
}

/**
 * htb_do_events - make mode changes to classes at the level
 *
663
 * Scans event queue for pending events and applies them. Returns time of
664
 * next pending event (0 for no event in pq, q->now for too many events).
665
 * Note: Applied are events whose have cl->pq_key <= q->now.
L
Linus Torvalds 已提交
666
 */
667 668
static psched_time_t htb_do_events(struct htb_sched *q, int level,
				   unsigned long start)
L
Linus Torvalds 已提交
669
{
670 671 672
	/* don't run for longer than 2 jiffies; 2 is used instead of
	   1 to simplify things when jiffy is going to be incremented
	   too soon */
673
	unsigned long stop_at = start + 2;
674
	while (time_before(jiffies, stop_at)) {
L
Linus Torvalds 已提交
675 676
		struct htb_class *cl;
		long diff;
677 678
		struct rb_node *p = rb_first(&q->wait_pq[level]);

S
Stephen Hemminger 已提交
679 680
		if (!p)
			return 0;
L
Linus Torvalds 已提交
681 682

		cl = rb_entry(p, struct htb_class, pq_node);
683 684 685
		if (cl->pq_key > q->now)
			return cl->pq_key;

S
Stephen Hemminger 已提交
686
		htb_safe_rb_erase(p, q->wait_pq + level);
687
		diff = psched_tdiff_bounded(q->now, cl->t_c, cl->mbuffer);
S
Stephen Hemminger 已提交
688
		htb_change_class_mode(q, cl, &diff);
L
Linus Torvalds 已提交
689
		if (cl->cmode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
690
			htb_add_to_wait_tree(q, cl, diff);
L
Linus Torvalds 已提交
691
	}
692 693

	/* too much load - let's continue after a break for scheduling */
694 695 696 697
	if (!(q->warned & HTB_WARN_TOOMANYEVENTS)) {
		printk(KERN_WARNING "htb: too many events!\n");
		q->warned |= HTB_WARN_TOOMANYEVENTS;
	}
698 699

	return q->now;
L
Linus Torvalds 已提交
700 701 702 703
}

/* Returns class->node+prio from id-tree where classe's id is >= id. NULL
   is no such one exists. */
S
Stephen Hemminger 已提交
704 705
static struct rb_node *htb_id_find_next_upper(int prio, struct rb_node *n,
					      u32 id)
L
Linus Torvalds 已提交
706 707 708
{
	struct rb_node *r = NULL;
	while (n) {
S
Stephen Hemminger 已提交
709 710 711
		struct htb_class *cl =
		    rb_entry(n, struct htb_class, node[prio]);

712
		if (id > cl->common.classid) {
L
Linus Torvalds 已提交
713
			n = n->rb_right;
714
		} else if (id < cl->common.classid) {
L
Linus Torvalds 已提交
715 716
			r = n;
			n = n->rb_left;
717 718
		} else {
			return n;
L
Linus Torvalds 已提交
719 720 721 722 723 724 725 726 727 728
		}
	}
	return r;
}

/**
 * htb_lookup_leaf - returns next leaf class in DRR order
 *
 * Find leaf where current feed pointers points to.
 */
S
Stephen Hemminger 已提交
729 730
static struct htb_class *htb_lookup_leaf(struct rb_root *tree, int prio,
					 struct rb_node **pptr, u32 * pid)
L
Linus Torvalds 已提交
731 732 733 734 735 736
{
	int i;
	struct {
		struct rb_node *root;
		struct rb_node **pptr;
		u32 *pid;
S
Stephen Hemminger 已提交
737 738
	} stk[TC_HTB_MAXDEPTH], *sp = stk;

739
	BUG_ON(!tree->rb_node);
L
Linus Torvalds 已提交
740 741 742 743 744
	sp->root = tree->rb_node;
	sp->pptr = pptr;
	sp->pid = pid;

	for (i = 0; i < 65535; i++) {
S
Stephen Hemminger 已提交
745
		if (!*sp->pptr && *sp->pid) {
746
			/* ptr was invalidated but id is valid - try to recover
L
Linus Torvalds 已提交
747
			   the original or next ptr */
S
Stephen Hemminger 已提交
748 749
			*sp->pptr =
			    htb_id_find_next_upper(prio, sp->root, *sp->pid);
L
Linus Torvalds 已提交
750
		}
S
Stephen Hemminger 已提交
751 752 753
		*sp->pid = 0;	/* ptr is valid now so that remove this hint as it
				   can become out of date quickly */
		if (!*sp->pptr) {	/* we are at right end; rewind & go up */
L
Linus Torvalds 已提交
754
			*sp->pptr = sp->root;
S
Stephen Hemminger 已提交
755
			while ((*sp->pptr)->rb_left)
L
Linus Torvalds 已提交
756 757 758
				*sp->pptr = (*sp->pptr)->rb_left;
			if (sp > stk) {
				sp--;
759 760
				if (!*sp->pptr) {
					WARN_ON(1);
S
Stephen Hemminger 已提交
761
					return NULL;
762
				}
S
Stephen Hemminger 已提交
763
				htb_next_rb_node(sp->pptr);
L
Linus Torvalds 已提交
764 765 766
			}
		} else {
			struct htb_class *cl;
S
Stephen Hemminger 已提交
767 768
			cl = rb_entry(*sp->pptr, struct htb_class, node[prio]);
			if (!cl->level)
L
Linus Torvalds 已提交
769 770
				return cl;
			(++sp)->root = cl->un.inner.feed[prio].rb_node;
S
Stephen Hemminger 已提交
771 772
			sp->pptr = cl->un.inner.ptr + prio;
			sp->pid = cl->un.inner.last_ptr_id + prio;
L
Linus Torvalds 已提交
773 774
		}
	}
775
	WARN_ON(1);
L
Linus Torvalds 已提交
776 777 778 779 780
	return NULL;
}

/* dequeues packet at given priority and level; call only if
   you are sure that there is active class at prio/level */
S
Stephen Hemminger 已提交
781 782
static struct sk_buff *htb_dequeue_tree(struct htb_sched *q, int prio,
					int level)
L
Linus Torvalds 已提交
783 784
{
	struct sk_buff *skb = NULL;
S
Stephen Hemminger 已提交
785
	struct htb_class *cl, *start;
L
Linus Torvalds 已提交
786
	/* look initial class up in the row */
S
Stephen Hemminger 已提交
787 788 789 790
	start = cl = htb_lookup_leaf(q->row[level] + prio, prio,
				     q->ptr[level] + prio,
				     q->last_ptr_id[level] + prio);

L
Linus Torvalds 已提交
791 792
	do {
next:
793
		if (unlikely(!cl))
S
Stephen Hemminger 已提交
794
			return NULL;
L
Linus Torvalds 已提交
795 796 797

		/* class can be empty - it is unlikely but can be true if leaf
		   qdisc drops packets in enqueue routine or if someone used
798
		   graft operation on the leaf since last dequeue;
L
Linus Torvalds 已提交
799 800 801
		   simply deactivate and skip such class */
		if (unlikely(cl->un.leaf.q->q.qlen == 0)) {
			struct htb_class *next;
S
Stephen Hemminger 已提交
802
			htb_deactivate(q, cl);
L
Linus Torvalds 已提交
803 804 805

			/* row/level might become empty */
			if ((q->row_mask[level] & (1 << prio)) == 0)
S
Stephen Hemminger 已提交
806
				return NULL;
L
Linus Torvalds 已提交
807

S
Stephen Hemminger 已提交
808 809 810 811 812
			next = htb_lookup_leaf(q->row[level] + prio,
					       prio, q->ptr[level] + prio,
					       q->last_ptr_id[level] + prio);

			if (cl == start)	/* fix start if we just deleted it */
L
Linus Torvalds 已提交
813 814 815 816
				start = next;
			cl = next;
			goto next;
		}
S
Stephen Hemminger 已提交
817 818 819

		skb = cl->un.leaf.q->dequeue(cl->un.leaf.q);
		if (likely(skb != NULL))
L
Linus Torvalds 已提交
820
			break;
821

822
		qdisc_warn_nonwc("htb", cl->un.leaf.q);
S
Stephen Hemminger 已提交
823 824 825 826 827
		htb_next_rb_node((level ? cl->parent->un.inner.ptr : q->
				  ptr[0]) + prio);
		cl = htb_lookup_leaf(q->row[level] + prio, prio,
				     q->ptr[level] + prio,
				     q->last_ptr_id[level] + prio);
L
Linus Torvalds 已提交
828 829 830 831

	} while (cl != start);

	if (likely(skb != NULL)) {
832 833
		cl->un.leaf.deficit[level] -= qdisc_pkt_len(skb);
		if (cl->un.leaf.deficit[level] < 0) {
834
			cl->un.leaf.deficit[level] += cl->quantum;
S
Stephen Hemminger 已提交
835 836
			htb_next_rb_node((level ? cl->parent->un.inner.ptr : q->
					  ptr[0]) + prio);
L
Linus Torvalds 已提交
837 838 839 840
		}
		/* this used to be after charge_class but this constelation
		   gives us slightly better performance */
		if (!cl->un.leaf.q->q.qlen)
S
Stephen Hemminger 已提交
841
			htb_deactivate(q, cl);
842
		htb_charge_class(q, cl, level, skb);
L
Linus Torvalds 已提交
843 844 845 846 847 848 849 850 851
	}
	return skb;
}

static struct sk_buff *htb_dequeue(struct Qdisc *sch)
{
	struct sk_buff *skb = NULL;
	struct htb_sched *q = qdisc_priv(sch);
	int level;
852
	psched_time_t next_event;
853
	unsigned long start_at;
L
Linus Torvalds 已提交
854 855

	/* try to dequeue direct packets as high prio (!) to minimize cpu work */
S
Stephen Hemminger 已提交
856 857
	skb = __skb_dequeue(&q->direct_queue);
	if (skb != NULL) {
L
Linus Torvalds 已提交
858 859 860 861 862
		sch->flags &= ~TCQ_F_THROTTLED;
		sch->q.qlen--;
		return skb;
	}

S
Stephen Hemminger 已提交
863 864
	if (!sch->q.qlen)
		goto fin;
865
	q->now = psched_get_time();
866
	start_at = jiffies;
L
Linus Torvalds 已提交
867

868
	next_event = q->now + 5 * PSCHED_TICKS_PER_SEC;
869

L
Linus Torvalds 已提交
870 871 872
	for (level = 0; level < TC_HTB_MAXDEPTH; level++) {
		/* common case optimization - skip event handler quickly */
		int m;
873 874 875
		psched_time_t event;

		if (q->now >= q->near_ev_cache[level]) {
876
			event = htb_do_events(q, level, start_at);
877 878 879
			if (!event)
				event = q->now + PSCHED_TICKS_PER_SEC;
			q->near_ev_cache[level] = event;
L
Linus Torvalds 已提交
880
		} else
881 882
			event = q->near_ev_cache[level];

883
		if (next_event > event)
884
			next_event = event;
S
Stephen Hemminger 已提交
885

L
Linus Torvalds 已提交
886 887
		m = ~q->row_mask[level];
		while (m != (int)(-1)) {
S
Stephen Hemminger 已提交
888
			int prio = ffz(m);
L
Linus Torvalds 已提交
889
			m |= 1 << prio;
S
Stephen Hemminger 已提交
890
			skb = htb_dequeue_tree(q, prio, level);
L
Linus Torvalds 已提交
891 892 893 894 895 896 897
			if (likely(skb != NULL)) {
				sch->q.qlen--;
				sch->flags &= ~TCQ_F_THROTTLED;
				goto fin;
			}
		}
	}
898
	sch->qstats.overlimits++;
899 900 901 902
	if (likely(next_event > q->now))
		qdisc_watchdog_schedule(&q->watchdog, next_event);
	else
		schedule_work(&q->work);
L
Linus Torvalds 已提交
903 904 905 906 907
fin:
	return skb;
}

/* try to drop from each class (by prio) until one succeed */
S
Stephen Hemminger 已提交
908
static unsigned int htb_drop(struct Qdisc *sch)
L
Linus Torvalds 已提交
909 910 911 912 913 914
{
	struct htb_sched *q = qdisc_priv(sch);
	int prio;

	for (prio = TC_HTB_NUMPRIO - 1; prio >= 0; prio--) {
		struct list_head *p;
S
Stephen Hemminger 已提交
915
		list_for_each(p, q->drops + prio) {
L
Linus Torvalds 已提交
916 917 918
			struct htb_class *cl = list_entry(p, struct htb_class,
							  un.leaf.drop_list);
			unsigned int len;
S
Stephen Hemminger 已提交
919 920
			if (cl->un.leaf.q->ops->drop &&
			    (len = cl->un.leaf.q->ops->drop(cl->un.leaf.q))) {
L
Linus Torvalds 已提交
921 922
				sch->q.qlen--;
				if (!cl->un.leaf.q->q.qlen)
S
Stephen Hemminger 已提交
923
					htb_deactivate(q, cl);
L
Linus Torvalds 已提交
924 925 926 927 928 929 930 931 932
				return len;
			}
		}
	}
	return 0;
}

/* reset all classes */
/* always caled under BH & queue lock */
S
Stephen Hemminger 已提交
933
static void htb_reset(struct Qdisc *sch)
L
Linus Torvalds 已提交
934 935
{
	struct htb_sched *q = qdisc_priv(sch);
936 937 938
	struct htb_class *cl;
	struct hlist_node *n;
	unsigned int i;
939

940 941
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode) {
L
Linus Torvalds 已提交
942
			if (cl->level)
S
Stephen Hemminger 已提交
943
				memset(&cl->un.inner, 0, sizeof(cl->un.inner));
L
Linus Torvalds 已提交
944
			else {
S
Stephen Hemminger 已提交
945
				if (cl->un.leaf.q)
L
Linus Torvalds 已提交
946 947 948 949 950 951 952 953
					qdisc_reset(cl->un.leaf.q);
				INIT_LIST_HEAD(&cl->un.leaf.drop_list);
			}
			cl->prio_activity = 0;
			cl->cmode = HTB_CAN_SEND;

		}
	}
954
	qdisc_watchdog_cancel(&q->watchdog);
L
Linus Torvalds 已提交
955 956
	__skb_queue_purge(&q->direct_queue);
	sch->q.qlen = 0;
S
Stephen Hemminger 已提交
957 958 959 960
	memset(q->row, 0, sizeof(q->row));
	memset(q->row_mask, 0, sizeof(q->row_mask));
	memset(q->wait_pq, 0, sizeof(q->wait_pq));
	memset(q->ptr, 0, sizeof(q->ptr));
L
Linus Torvalds 已提交
961
	for (i = 0; i < TC_HTB_NUMPRIO; i++)
S
Stephen Hemminger 已提交
962
		INIT_LIST_HEAD(q->drops + i);
L
Linus Torvalds 已提交
963 964
}

965 966 967 968 969 970 971
static const struct nla_policy htb_policy[TCA_HTB_MAX + 1] = {
	[TCA_HTB_PARMS]	= { .len = sizeof(struct tc_htb_opt) },
	[TCA_HTB_INIT]	= { .len = sizeof(struct tc_htb_glob) },
	[TCA_HTB_CTAB]	= { .type = NLA_BINARY, .len = TC_RTAB_SIZE },
	[TCA_HTB_RTAB]	= { .type = NLA_BINARY, .len = TC_RTAB_SIZE },
};

972 973 974 975 976 977 978 979
static void htb_work_func(struct work_struct *work)
{
	struct htb_sched *q = container_of(work, struct htb_sched, work);
	struct Qdisc *sch = q->watchdog.qdisc;

	__netif_schedule(qdisc_root(sch));
}

980
static int htb_init(struct Qdisc *sch, struct nlattr *opt)
L
Linus Torvalds 已提交
981 982
{
	struct htb_sched *q = qdisc_priv(sch);
983
	struct nlattr *tb[TCA_HTB_INIT + 1];
L
Linus Torvalds 已提交
984
	struct tc_htb_glob *gopt;
985
	int err;
L
Linus Torvalds 已提交
986
	int i;
987 988 989 990

	if (!opt)
		return -EINVAL;

991
	err = nla_parse_nested(tb, TCA_HTB_INIT, opt, htb_policy);
992 993 994
	if (err < 0)
		return err;

995
	if (tb[TCA_HTB_INIT] == NULL) {
L
Linus Torvalds 已提交
996 997 998
		printk(KERN_ERR "HTB: hey probably you have bad tc tool ?\n");
		return -EINVAL;
	}
999
	gopt = nla_data(tb[TCA_HTB_INIT]);
L
Linus Torvalds 已提交
1000
	if (gopt->version != HTB_VER >> 16) {
S
Stephen Hemminger 已提交
1001 1002 1003
		printk(KERN_ERR
		       "HTB: need tc/htb version %d (minor is %d), you have %d\n",
		       HTB_VER >> 16, HTB_VER & 0xffff, gopt->version);
L
Linus Torvalds 已提交
1004 1005 1006
		return -EINVAL;
	}

1007 1008 1009
	err = qdisc_class_hash_init(&q->clhash);
	if (err < 0)
		return err;
L
Linus Torvalds 已提交
1010
	for (i = 0; i < TC_HTB_NUMPRIO; i++)
S
Stephen Hemminger 已提交
1011
		INIT_LIST_HEAD(q->drops + i);
L
Linus Torvalds 已提交
1012

1013
	qdisc_watchdog_init(&q->watchdog, sch);
1014
	INIT_WORK(&q->work, htb_work_func);
L
Linus Torvalds 已提交
1015 1016
	skb_queue_head_init(&q->direct_queue);

1017
	q->direct_qlen = qdisc_dev(sch)->tx_queue_len;
S
Stephen Hemminger 已提交
1018
	if (q->direct_qlen < 2)	/* some devices have zero tx_queue_len */
L
Linus Torvalds 已提交
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
		q->direct_qlen = 2;

	if ((q->rate2quantum = gopt->rate2quantum) < 1)
		q->rate2quantum = 1;
	q->defcls = gopt->defcls;

	return 0;
}

static int htb_dump(struct Qdisc *sch, struct sk_buff *skb)
{
1030
	spinlock_t *root_lock = qdisc_root_sleeping_lock(sch);
L
Linus Torvalds 已提交
1031
	struct htb_sched *q = qdisc_priv(sch);
1032
	struct nlattr *nest;
L
Linus Torvalds 已提交
1033
	struct tc_htb_glob gopt;
1034

1035
	spin_lock_bh(root_lock);
L
Linus Torvalds 已提交
1036

1037
	gopt.direct_pkts = q->direct_pkts;
L
Linus Torvalds 已提交
1038 1039 1040
	gopt.version = HTB_VER;
	gopt.rate2quantum = q->rate2quantum;
	gopt.defcls = q->defcls;
1041
	gopt.debug = 0;
1042 1043 1044 1045

	nest = nla_nest_start(skb, TCA_OPTIONS);
	if (nest == NULL)
		goto nla_put_failure;
1046
	NLA_PUT(skb, TCA_HTB_INIT, sizeof(gopt), &gopt);
1047 1048
	nla_nest_end(skb, nest);

1049
	spin_unlock_bh(root_lock);
L
Linus Torvalds 已提交
1050
	return skb->len;
1051

1052
nla_put_failure:
1053
	spin_unlock_bh(root_lock);
1054
	nla_nest_cancel(skb, nest);
L
Linus Torvalds 已提交
1055 1056 1057 1058
	return -1;
}

static int htb_dump_class(struct Qdisc *sch, unsigned long arg,
S
Stephen Hemminger 已提交
1059
			  struct sk_buff *skb, struct tcmsg *tcm)
L
Linus Torvalds 已提交
1060
{
S
Stephen Hemminger 已提交
1061
	struct htb_class *cl = (struct htb_class *)arg;
1062
	spinlock_t *root_lock = qdisc_root_sleeping_lock(sch);
1063
	struct nlattr *nest;
L
Linus Torvalds 已提交
1064 1065
	struct tc_htb_opt opt;

1066
	spin_lock_bh(root_lock);
1067 1068
	tcm->tcm_parent = cl->parent ? cl->parent->common.classid : TC_H_ROOT;
	tcm->tcm_handle = cl->common.classid;
L
Linus Torvalds 已提交
1069 1070 1071
	if (!cl->level && cl->un.leaf.q)
		tcm->tcm_info = cl->un.leaf.q->handle;

1072 1073 1074
	nest = nla_nest_start(skb, TCA_OPTIONS);
	if (nest == NULL)
		goto nla_put_failure;
L
Linus Torvalds 已提交
1075

S
Stephen Hemminger 已提交
1076
	memset(&opt, 0, sizeof(opt));
L
Linus Torvalds 已提交
1077

S
Stephen Hemminger 已提交
1078 1079 1080 1081
	opt.rate = cl->rate->rate;
	opt.buffer = cl->buffer;
	opt.ceil = cl->ceil->rate;
	opt.cbuffer = cl->cbuffer;
1082 1083
	opt.quantum = cl->quantum;
	opt.prio = cl->prio;
S
Stephen Hemminger 已提交
1084
	opt.level = cl->level;
1085
	NLA_PUT(skb, TCA_HTB_PARMS, sizeof(opt), &opt);
1086 1087

	nla_nest_end(skb, nest);
1088
	spin_unlock_bh(root_lock);
L
Linus Torvalds 已提交
1089
	return skb->len;
1090

1091
nla_put_failure:
1092
	spin_unlock_bh(root_lock);
1093
	nla_nest_cancel(skb, nest);
L
Linus Torvalds 已提交
1094 1095 1096 1097
	return -1;
}

static int
S
Stephen Hemminger 已提交
1098
htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
L
Linus Torvalds 已提交
1099
{
S
Stephen Hemminger 已提交
1100
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115

	if (!cl->level && cl->un.leaf.q)
		cl->qstats.qlen = cl->un.leaf.q->q.qlen;
	cl->xstats.tokens = cl->tokens;
	cl->xstats.ctokens = cl->ctokens;

	if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
	    gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 ||
	    gnet_stats_copy_queue(d, &cl->qstats) < 0)
		return -1;

	return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));
}

static int htb_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
S
Stephen Hemminger 已提交
1116
		     struct Qdisc **old)
L
Linus Torvalds 已提交
1117
{
S
Stephen Hemminger 已提交
1118
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1119 1120

	if (cl && !cl->level) {
1121
		if (new == NULL &&
1122
		    (new = qdisc_create_dflt(qdisc_dev(sch), sch->dev_queue,
1123
					     &pfifo_qdisc_ops,
1124
					     cl->common.classid))
S
Stephen Hemminger 已提交
1125 1126
		    == NULL)
			return -ENOBUFS;
L
Linus Torvalds 已提交
1127
		sch_tree_lock(sch);
1128 1129 1130
		*old = cl->un.leaf.q;
		cl->un.leaf.q = new;
		if (*old != NULL) {
1131
			qdisc_tree_decrease_qlen(*old, (*old)->q.qlen);
L
Linus Torvalds 已提交
1132 1133 1134 1135 1136 1137 1138 1139
			qdisc_reset(*old);
		}
		sch_tree_unlock(sch);
		return 0;
	}
	return -ENOENT;
}

S
Stephen Hemminger 已提交
1140
static struct Qdisc *htb_leaf(struct Qdisc *sch, unsigned long arg)
L
Linus Torvalds 已提交
1141
{
S
Stephen Hemminger 已提交
1142
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1143 1144 1145
	return (cl && !cl->level) ? cl->un.leaf.q : NULL;
}

1146 1147 1148 1149 1150 1151 1152 1153
static void htb_qlen_notify(struct Qdisc *sch, unsigned long arg)
{
	struct htb_class *cl = (struct htb_class *)arg;

	if (cl->un.leaf.q->q.qlen == 0)
		htb_deactivate(qdisc_priv(sch), cl);
}

L
Linus Torvalds 已提交
1154 1155
static unsigned long htb_get(struct Qdisc *sch, u32 classid)
{
S
Stephen Hemminger 已提交
1156 1157
	struct htb_class *cl = htb_find(classid, sch);
	if (cl)
L
Linus Torvalds 已提交
1158 1159 1160 1161
		cl->refcnt++;
	return (unsigned long)cl;
}

1162 1163 1164 1165 1166
static inline int htb_parent_last_child(struct htb_class *cl)
{
	if (!cl->parent)
		/* the root class */
		return 0;
1167
	if (cl->parent->children > 1)
1168 1169 1170 1171 1172
		/* not the last child */
		return 0;
	return 1;
}

1173 1174
static void htb_parent_to_leaf(struct htb_sched *q, struct htb_class *cl,
			       struct Qdisc *new_q)
1175 1176 1177
{
	struct htb_class *parent = cl->parent;

1178
	WARN_ON(cl->level || !cl->un.leaf.q || cl->prio_activity);
1179

1180 1181 1182
	if (parent->cmode != HTB_CAN_SEND)
		htb_safe_rb_erase(&parent->pq_node, q->wait_pq + parent->level);

1183 1184 1185 1186 1187 1188
	parent->level = 0;
	memset(&parent->un.inner, 0, sizeof(parent->un.inner));
	INIT_LIST_HEAD(&parent->un.leaf.drop_list);
	parent->un.leaf.q = new_q ? new_q : &noop_qdisc;
	parent->tokens = parent->buffer;
	parent->ctokens = parent->cbuffer;
1189
	parent->t_c = psched_get_time();
1190 1191 1192
	parent->cmode = HTB_CAN_SEND;
}

S
Stephen Hemminger 已提交
1193
static void htb_destroy_class(struct Qdisc *sch, struct htb_class *cl)
L
Linus Torvalds 已提交
1194 1195
{
	if (!cl->level) {
1196
		WARN_ON(!cl->un.leaf.q);
L
Linus Torvalds 已提交
1197 1198
		qdisc_destroy(cl->un.leaf.q);
	}
1199
	gen_kill_estimator(&cl->bstats, &cl->rate_est);
L
Linus Torvalds 已提交
1200 1201
	qdisc_put_rtab(cl->rate);
	qdisc_put_rtab(cl->ceil);
S
Stephen Hemminger 已提交
1202

1203
	tcf_destroy_chain(&cl->filter_list);
L
Linus Torvalds 已提交
1204 1205 1206
	kfree(cl);
}

S
Stephen Hemminger 已提交
1207
static void htb_destroy(struct Qdisc *sch)
L
Linus Torvalds 已提交
1208 1209
{
	struct htb_sched *q = qdisc_priv(sch);
1210 1211 1212
	struct hlist_node *n, *next;
	struct htb_class *cl;
	unsigned int i;
L
Linus Torvalds 已提交
1213

1214
	cancel_work_sync(&q->work);
1215
	qdisc_watchdog_cancel(&q->watchdog);
L
Linus Torvalds 已提交
1216
	/* This line used to be after htb_destroy_class call below
1217
	   and surprisingly it worked in 2.4. But it must precede it
L
Linus Torvalds 已提交
1218 1219
	   because filter need its target class alive to be able to call
	   unbind_filter on it (without Oops). */
1220
	tcf_destroy_chain(&q->filter_list);
S
Stephen Hemminger 已提交
1221

1222 1223
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode)
1224 1225
			tcf_destroy_chain(&cl->filter_list);
	}
1226 1227 1228
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry_safe(cl, n, next, &q->clhash.hash[i],
					  common.hnode)
1229 1230
			htb_destroy_class(sch, cl);
	}
1231
	qdisc_class_hash_destroy(&q->clhash);
L
Linus Torvalds 已提交
1232 1233 1234 1235 1236 1237
	__skb_queue_purge(&q->direct_queue);
}

static int htb_delete(struct Qdisc *sch, unsigned long arg)
{
	struct htb_sched *q = qdisc_priv(sch);
S
Stephen Hemminger 已提交
1238
	struct htb_class *cl = (struct htb_class *)arg;
1239
	unsigned int qlen;
1240 1241
	struct Qdisc *new_q = NULL;
	int last_child = 0;
L
Linus Torvalds 已提交
1242 1243 1244 1245

	// TODO: why don't allow to delete subtree ? references ? does
	// tc subsys quarantee us that in htb_destroy it holds no class
	// refs so that we can remove children safely there ?
1246
	if (cl->children || cl->filter_cnt)
L
Linus Torvalds 已提交
1247
		return -EBUSY;
S
Stephen Hemminger 已提交
1248

1249
	if (!cl->level && htb_parent_last_child(cl)) {
1250
		new_q = qdisc_create_dflt(qdisc_dev(sch), sch->dev_queue,
1251 1252
					  &pfifo_qdisc_ops,
					  cl->parent->common.classid);
1253 1254 1255
		last_child = 1;
	}

L
Linus Torvalds 已提交
1256
	sch_tree_lock(sch);
S
Stephen Hemminger 已提交
1257

1258
	if (!cl->level) {
1259
		qlen = cl->un.leaf.q->q.qlen;
1260
		qdisc_reset(cl->un.leaf.q);
1261
		qdisc_tree_decrease_qlen(cl->un.leaf.q, qlen);
1262 1263
	}

1264 1265
	/* delete from hash and active; remainder in destroy_class */
	qdisc_class_hash_remove(&q->clhash, &cl->common);
1266 1267
	if (cl->parent)
		cl->parent->children--;
1268

L
Linus Torvalds 已提交
1269
	if (cl->prio_activity)
S
Stephen Hemminger 已提交
1270
		htb_deactivate(q, cl);
L
Linus Torvalds 已提交
1271

1272 1273 1274
	if (cl->cmode != HTB_CAN_SEND)
		htb_safe_rb_erase(&cl->pq_node, q->wait_pq + cl->level);

1275
	if (last_child)
1276
		htb_parent_to_leaf(q, cl, new_q);
1277

1278 1279 1280 1281 1282
	BUG_ON(--cl->refcnt == 0);
	/*
	 * This shouldn't happen: we "hold" one cops->get() when called
	 * from tc_ctl_tclass; the destroy method is done from cops->put().
	 */
L
Linus Torvalds 已提交
1283 1284 1285 1286 1287 1288 1289

	sch_tree_unlock(sch);
	return 0;
}

static void htb_put(struct Qdisc *sch, unsigned long arg)
{
S
Stephen Hemminger 已提交
1290
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1291 1292

	if (--cl->refcnt == 0)
S
Stephen Hemminger 已提交
1293
		htb_destroy_class(sch, cl);
L
Linus Torvalds 已提交
1294 1295
}

S
Stephen Hemminger 已提交
1296
static int htb_change_class(struct Qdisc *sch, u32 classid,
1297
			    u32 parentid, struct nlattr **tca,
S
Stephen Hemminger 已提交
1298
			    unsigned long *arg)
L
Linus Torvalds 已提交
1299 1300 1301
{
	int err = -EINVAL;
	struct htb_sched *q = qdisc_priv(sch);
S
Stephen Hemminger 已提交
1302
	struct htb_class *cl = (struct htb_class *)*arg, *parent;
1303
	struct nlattr *opt = tca[TCA_OPTIONS];
L
Linus Torvalds 已提交
1304
	struct qdisc_rate_table *rtab = NULL, *ctab = NULL;
1305
	struct nlattr *tb[TCA_HTB_RTAB + 1];
L
Linus Torvalds 已提交
1306 1307 1308
	struct tc_htb_opt *hopt;

	/* extract all subattrs from opt attr */
1309 1310 1311
	if (!opt)
		goto failure;

1312
	err = nla_parse_nested(tb, TCA_HTB_RTAB, opt, htb_policy);
1313 1314 1315 1316
	if (err < 0)
		goto failure;

	err = -EINVAL;
1317
	if (tb[TCA_HTB_PARMS] == NULL)
L
Linus Torvalds 已提交
1318 1319
		goto failure;

S
Stephen Hemminger 已提交
1320 1321
	parent = parentid == TC_H_ROOT ? NULL : htb_find(parentid, sch);

1322
	hopt = nla_data(tb[TCA_HTB_PARMS]);
1323

1324 1325
	rtab = qdisc_get_rtab(&hopt->rate, tb[TCA_HTB_RTAB]);
	ctab = qdisc_get_rtab(&hopt->ceil, tb[TCA_HTB_CTAB]);
S
Stephen Hemminger 已提交
1326 1327
	if (!rtab || !ctab)
		goto failure;
L
Linus Torvalds 已提交
1328

S
Stephen Hemminger 已提交
1329
	if (!cl) {		/* new class */
L
Linus Torvalds 已提交
1330
		struct Qdisc *new_q;
S
Stephen Hemminger 已提交
1331
		int prio;
1332
		struct {
1333
			struct nlattr		nla;
1334 1335
			struct gnet_estimator	opt;
		} est = {
1336 1337 1338
			.nla = {
				.nla_len	= nla_attr_size(sizeof(est.opt)),
				.nla_type	= TCA_RATE,
1339 1340 1341 1342 1343 1344 1345
			},
			.opt = {
				/* 4s interval, 16s averaging constant */
				.interval	= 2,
				.ewma_log	= 2,
			},
		};
S
Stephen Hemminger 已提交
1346

L
Linus Torvalds 已提交
1347
		/* check for valid classid */
S
Stephen Hemminger 已提交
1348 1349
		if (!classid || TC_H_MAJ(classid ^ sch->handle)
		    || htb_find(classid, sch))
L
Linus Torvalds 已提交
1350 1351 1352 1353 1354 1355 1356 1357
			goto failure;

		/* check maximal depth */
		if (parent && parent->parent && parent->parent->level < 2) {
			printk(KERN_ERR "htb: tree is too deep\n");
			goto failure;
		}
		err = -ENOBUFS;
1358
		if ((cl = kzalloc(sizeof(*cl), GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
1359
			goto failure;
S
Stephen Hemminger 已提交
1360

1361 1362 1363 1364 1365 1366 1367 1368
		err = gen_new_estimator(&cl->bstats, &cl->rate_est,
					qdisc_root_sleeping_lock(sch),
					tca[TCA_RATE] ? : &est.nla);
		if (err) {
			kfree(cl);
			goto failure;
		}

L
Linus Torvalds 已提交
1369
		cl->refcnt = 1;
1370
		cl->children = 0;
L
Linus Torvalds 已提交
1371
		INIT_LIST_HEAD(&cl->un.leaf.drop_list);
S
Stephen Hemminger 已提交
1372 1373 1374 1375
		RB_CLEAR_NODE(&cl->pq_node);

		for (prio = 0; prio < TC_HTB_NUMPRIO; prio++)
			RB_CLEAR_NODE(&cl->node[prio]);
L
Linus Torvalds 已提交
1376 1377 1378 1379

		/* create leaf qdisc early because it uses kmalloc(GFP_KERNEL)
		   so that can't be used inside of sch_tree_lock
		   -- thanks to Karlis Peisenieks */
1380
		new_q = qdisc_create_dflt(qdisc_dev(sch), sch->dev_queue,
1381
					  &pfifo_qdisc_ops, classid);
L
Linus Torvalds 已提交
1382 1383
		sch_tree_lock(sch);
		if (parent && !parent->level) {
1384 1385
			unsigned int qlen = parent->un.leaf.q->q.qlen;

L
Linus Torvalds 已提交
1386
			/* turn parent into inner node */
1387 1388
			qdisc_reset(parent->un.leaf.q);
			qdisc_tree_decrease_qlen(parent->un.leaf.q, qlen);
S
Stephen Hemminger 已提交
1389 1390 1391
			qdisc_destroy(parent->un.leaf.q);
			if (parent->prio_activity)
				htb_deactivate(q, parent);
L
Linus Torvalds 已提交
1392 1393 1394

			/* remove from evt list because of level change */
			if (parent->cmode != HTB_CAN_SEND) {
S
Stephen Hemminger 已提交
1395
				htb_safe_rb_erase(&parent->pq_node, q->wait_pq);
L
Linus Torvalds 已提交
1396 1397 1398
				parent->cmode = HTB_CAN_SEND;
			}
			parent->level = (parent->parent ? parent->parent->level
S
Stephen Hemminger 已提交
1399 1400
					 : TC_HTB_MAXDEPTH) - 1;
			memset(&parent->un.inner, 0, sizeof(parent->un.inner));
L
Linus Torvalds 已提交
1401 1402 1403 1404
		}
		/* leaf (we) needs elementary qdisc */
		cl->un.leaf.q = new_q ? new_q : &noop_qdisc;

1405
		cl->common.classid = classid;
S
Stephen Hemminger 已提交
1406
		cl->parent = parent;
L
Linus Torvalds 已提交
1407 1408 1409 1410

		/* set class to be in HTB_CAN_SEND state */
		cl->tokens = hopt->buffer;
		cl->ctokens = hopt->cbuffer;
1411
		cl->mbuffer = 60 * PSCHED_TICKS_PER_SEC;	/* 1min */
1412
		cl->t_c = psched_get_time();
L
Linus Torvalds 已提交
1413 1414 1415
		cl->cmode = HTB_CAN_SEND;

		/* attach to the hash list and parent's family */
1416
		qdisc_class_hash_insert(&q->clhash, &cl->common);
1417 1418
		if (parent)
			parent->children++;
1419
	} else {
1420 1421 1422 1423 1424 1425 1426
		if (tca[TCA_RATE]) {
			err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
						    qdisc_root_sleeping_lock(sch),
						    tca[TCA_RATE]);
			if (err)
				return err;
		}
S
Stephen Hemminger 已提交
1427
		sch_tree_lock(sch);
1428
	}
L
Linus Torvalds 已提交
1429 1430

	/* it used to be a nasty bug here, we have to check that node
S
Stephen Hemminger 已提交
1431
	   is really leaf before changing cl->un.leaf ! */
L
Linus Torvalds 已提交
1432
	if (!cl->level) {
1433 1434
		cl->quantum = rtab->rate.rate / q->rate2quantum;
		if (!hopt->quantum && cl->quantum < 1000) {
S
Stephen Hemminger 已提交
1435 1436
			printk(KERN_WARNING
			       "HTB: quantum of class %X is small. Consider r2q change.\n",
1437
			       cl->common.classid);
1438
			cl->quantum = 1000;
L
Linus Torvalds 已提交
1439
		}
1440
		if (!hopt->quantum && cl->quantum > 200000) {
S
Stephen Hemminger 已提交
1441 1442
			printk(KERN_WARNING
			       "HTB: quantum of class %X is big. Consider r2q change.\n",
1443
			       cl->common.classid);
1444
			cl->quantum = 200000;
L
Linus Torvalds 已提交
1445 1446
		}
		if (hopt->quantum)
1447 1448 1449
			cl->quantum = hopt->quantum;
		if ((cl->prio = hopt->prio) >= TC_HTB_NUMPRIO)
			cl->prio = TC_HTB_NUMPRIO - 1;
L
Linus Torvalds 已提交
1450 1451 1452 1453
	}

	cl->buffer = hopt->buffer;
	cl->cbuffer = hopt->cbuffer;
S
Stephen Hemminger 已提交
1454 1455 1456 1457 1458 1459
	if (cl->rate)
		qdisc_put_rtab(cl->rate);
	cl->rate = rtab;
	if (cl->ceil)
		qdisc_put_rtab(cl->ceil);
	cl->ceil = ctab;
L
Linus Torvalds 已提交
1460 1461
	sch_tree_unlock(sch);

1462 1463
	qdisc_class_hash_grow(sch, &q->clhash);

L
Linus Torvalds 已提交
1464 1465 1466 1467
	*arg = (unsigned long)cl;
	return 0;

failure:
S
Stephen Hemminger 已提交
1468 1469 1470 1471
	if (rtab)
		qdisc_put_rtab(rtab);
	if (ctab)
		qdisc_put_rtab(ctab);
L
Linus Torvalds 已提交
1472 1473 1474 1475 1476 1477 1478 1479
	return err;
}

static struct tcf_proto **htb_find_tcf(struct Qdisc *sch, unsigned long arg)
{
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl = (struct htb_class *)arg;
	struct tcf_proto **fl = cl ? &cl->filter_list : &q->filter_list;
1480

L
Linus Torvalds 已提交
1481 1482 1483 1484
	return fl;
}

static unsigned long htb_bind_filter(struct Qdisc *sch, unsigned long parent,
S
Stephen Hemminger 已提交
1485
				     u32 classid)
L
Linus Torvalds 已提交
1486
{
S
Stephen Hemminger 已提交
1487
	struct htb_class *cl = htb_find(classid, sch);
1488

L
Linus Torvalds 已提交
1489
	/*if (cl && !cl->level) return 0;
S
Stephen Hemminger 已提交
1490 1491 1492 1493 1494 1495 1496
	   The line above used to be there to prevent attaching filters to
	   leaves. But at least tc_index filter uses this just to get class
	   for other reasons so that we have to allow for it.
	   ----
	   19.6.2002 As Werner explained it is ok - bind filter is just
	   another way to "lock" the class - unlike "get" this lock can
	   be broken by class during destroy IIUC.
L
Linus Torvalds 已提交
1497
	 */
S
Stephen Hemminger 已提交
1498 1499
	if (cl)
		cl->filter_cnt++;
L
Linus Torvalds 已提交
1500 1501 1502 1503 1504 1505
	return (unsigned long)cl;
}

static void htb_unbind_filter(struct Qdisc *sch, unsigned long arg)
{
	struct htb_class *cl = (struct htb_class *)arg;
1506

S
Stephen Hemminger 已提交
1507 1508
	if (cl)
		cl->filter_cnt--;
L
Linus Torvalds 已提交
1509 1510 1511 1512 1513
}

static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
	struct htb_sched *q = qdisc_priv(sch);
1514 1515 1516
	struct htb_class *cl;
	struct hlist_node *n;
	unsigned int i;
L
Linus Torvalds 已提交
1517 1518 1519 1520

	if (arg->stop)
		return;

1521 1522
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode) {
L
Linus Torvalds 已提交
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
			if (arg->count < arg->skip) {
				arg->count++;
				continue;
			}
			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
				arg->stop = 1;
				return;
			}
			arg->count++;
		}
	}
}

1536
static const struct Qdisc_class_ops htb_class_ops = {
L
Linus Torvalds 已提交
1537 1538
	.graft		=	htb_graft,
	.leaf		=	htb_leaf,
1539
	.qlen_notify	=	htb_qlen_notify,
L
Linus Torvalds 已提交
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551
	.get		=	htb_get,
	.put		=	htb_put,
	.change		=	htb_change_class,
	.delete		=	htb_delete,
	.walk		=	htb_walk,
	.tcf_chain	=	htb_find_tcf,
	.bind_tcf	=	htb_bind_filter,
	.unbind_tcf	=	htb_unbind_filter,
	.dump		=	htb_dump_class,
	.dump_stats	=	htb_dump_class_stats,
};

1552
static struct Qdisc_ops htb_qdisc_ops __read_mostly = {
L
Linus Torvalds 已提交
1553 1554 1555 1556 1557 1558
	.next		=	NULL,
	.cl_ops		=	&htb_class_ops,
	.id		=	"htb",
	.priv_size	=	sizeof(struct htb_sched),
	.enqueue	=	htb_enqueue,
	.dequeue	=	htb_dequeue,
1559
	.peek		=	qdisc_peek_dequeued,
L
Linus Torvalds 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	.drop		=	htb_drop,
	.init		=	htb_init,
	.reset		=	htb_reset,
	.destroy	=	htb_destroy,
	.change		=	NULL /* htb_change */,
	.dump		=	htb_dump,
	.owner		=	THIS_MODULE,
};

static int __init htb_module_init(void)
{
S
Stephen Hemminger 已提交
1571
	return register_qdisc(&htb_qdisc_ops);
L
Linus Torvalds 已提交
1572
}
S
Stephen Hemminger 已提交
1573
static void __exit htb_module_exit(void)
L
Linus Torvalds 已提交
1574
{
S
Stephen Hemminger 已提交
1575
	unregister_qdisc(&htb_qdisc_ops);
L
Linus Torvalds 已提交
1576
}
S
Stephen Hemminger 已提交
1577

L
Linus Torvalds 已提交
1578 1579 1580
module_init(htb_module_init)
module_exit(htb_module_exit)
MODULE_LICENSE("GPL");