sch_htb.c 42.6 KB
Newer Older
S
Stephen Hemminger 已提交
1
/*
L
Linus Torvalds 已提交
2 3 4 5 6 7 8 9 10 11 12 13
 * net/sched/sch_htb.c	Hierarchical token bucket, feed tree version
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 * Authors:	Martin Devera, <devik@cdi.cz>
 *
 * Credits (in time order) for older HTB versions:
 *              Stef Coene <stef.coene@docum.org>
 *			HTB support at LARTC mailing list
14
 *		Ondrej Kraus, <krauso@barr.cz>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21 22 23 24 25 26 27 28
 *			found missing INIT_QDISC(htb)
 *		Vladimir Smelhaus, Aamer Akhter, Bert Hubert
 *			helped a lot to locate nasty class stall bug
 *		Andi Kleen, Jamal Hadi, Bert Hubert
 *			code review and helpful comments on shaping
 *		Tomasz Wrona, <tw@eter.tym.pl>
 *			created test case so that I was able to fix nasty bug
 *		Wilfried Weissmann
 *			spotted bug in dequeue code and helped with fix
 *		Jiri Fojtasek
 *			fixed requeue routine
 *		and many others. thanks.
 */
#include <linux/module.h>
29
#include <linux/moduleparam.h>
L
Linus Torvalds 已提交
30 31 32 33 34 35 36
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/list.h>
#include <linux/compiler.h>
37
#include <linux/rbtree.h>
38
#include <linux/workqueue.h>
39
#include <linux/slab.h>
40
#include <net/netlink.h>
L
Linus Torvalds 已提交
41 42 43 44 45 46
#include <net/pkt_sched.h>

/* HTB algorithm.
    Author: devik@cdi.cz
    ========================================================================
    HTB is like TBF with multiple classes. It is also similar to CBQ because
47
    it allows to assign priority to each class in hierarchy.
L
Linus Torvalds 已提交
48 49 50
    In fact it is another implementation of Floyd's formal sharing.

    Levels:
51
    Each class is assigned level. Leaf has ALWAYS level 0 and root
L
Linus Torvalds 已提交
52 53 54 55
    classes have level TC_HTB_MAXDEPTH-1. Interior nodes has level
    one less than their parent.
*/

56
static int htb_hysteresis __read_mostly = 0; /* whether to use mode hysteresis for speedup */
S
Stephen Hemminger 已提交
57
#define HTB_VER 0x30011		/* major must be matched with number suplied by TC as version */
L
Linus Torvalds 已提交
58 59 60 61 62

#if HTB_VER >> 16 != TC_HTB_PROTOVER
#error "Mismatched sch_htb.c and pkt_sch.h"
#endif

63 64 65 66
/* Module parameter and sysfs export */
module_param    (htb_hysteresis, int, 0640);
MODULE_PARM_DESC(htb_hysteresis, "Hysteresis mode, less CPU load, less accurate");

L
Linus Torvalds 已提交
67 68
/* used internaly to keep status of single class */
enum htb_cmode {
S
Stephen Hemminger 已提交
69 70 71
	HTB_CANT_SEND,		/* class can't send and can't borrow */
	HTB_MAY_BORROW,		/* class can't send but may borrow */
	HTB_CAN_SEND		/* class can send */
L
Linus Torvalds 已提交
72 73
};

V
Vimalkumar 已提交
74 75 76 77 78 79
struct htb_rate_cfg {
	u64 rate_bps;
	u32 mult;
	u32 shift;
};

L
Linus Torvalds 已提交
80
/* interior & leaf nodes; props specific to leaves are marked L: */
S
Stephen Hemminger 已提交
81
struct htb_class {
82
	struct Qdisc_class_common common;
S
Stephen Hemminger 已提交
83
	/* general class parameters */
84
	struct gnet_stats_basic_packed bstats;
S
Stephen Hemminger 已提交
85 86 87 88
	struct gnet_stats_queue qstats;
	struct gnet_stats_rate_est rate_est;
	struct tc_htb_xstats xstats;	/* our special stats */
	int refcnt;		/* usage count of this class */
L
Linus Torvalds 已提交
89

S
Stephen Hemminger 已提交
90 91
	/* topology */
	int level;		/* our level (see above) */
92
	unsigned int children;
S
Stephen Hemminger 已提交
93 94
	struct htb_class *parent;	/* parent class */

95 96 97
	int prio;		/* these two are used only by leaves... */
	int quantum;		/* but stored for parent-to-leaf return */

S
Stephen Hemminger 已提交
98 99 100 101 102 103 104 105 106 107
	union {
		struct htb_class_leaf {
			struct Qdisc *q;
			int deficit[TC_HTB_MAXDEPTH];
			struct list_head drop_list;
		} leaf;
		struct htb_class_inner {
			struct rb_root feed[TC_HTB_NUMPRIO];	/* feed trees */
			struct rb_node *ptr[TC_HTB_NUMPRIO];	/* current class ptr */
			/* When class changes from state 1->2 and disconnects from
E
Eric Dumazet 已提交
108 109 110 111
			 * parent's feed then we lost ptr value and start from the
			 * first child again. Here we store classid of the
			 * last valid ptr (used when ptr is NULL).
			 */
S
Stephen Hemminger 已提交
112 113 114 115 116
			u32 last_ptr_id[TC_HTB_NUMPRIO];
		} inner;
	} un;
	struct rb_node node[TC_HTB_NUMPRIO];	/* node for self or feed tree */
	struct rb_node pq_node;	/* node for event queue */
117
	psched_time_t pq_key;
S
Stephen Hemminger 已提交
118 119 120 121 122 123 124 125 126

	int prio_activity;	/* for which prios are we active */
	enum htb_cmode cmode;	/* current mode of the class */

	/* class attached filters */
	struct tcf_proto *filter_list;
	int filter_cnt;

	/* token bucket parameters */
V
Vimalkumar 已提交
127 128 129
	struct htb_rate_cfg rate;
	struct htb_rate_cfg ceil;
	s64 buffer, cbuffer;	/* token bucket depth/rate */
S
Stephen Hemminger 已提交
130
	psched_tdiff_t mbuffer;	/* max wait time */
V
Vimalkumar 已提交
131
	s64 tokens, ctokens;	/* current number of tokens */
S
Stephen Hemminger 已提交
132
	psched_time_t t_c;	/* checkpoint time */
L
Linus Torvalds 已提交
133 134
};

S
Stephen Hemminger 已提交
135
struct htb_sched {
136
	struct Qdisc_class_hash clhash;
137
	struct list_head drops[TC_HTB_NUMPRIO];/* active leaves (for drops) */
S
Stephen Hemminger 已提交
138 139 140 141 142 143

	/* self list - roots of self generating tree */
	struct rb_root row[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
	int row_mask[TC_HTB_MAXDEPTH];
	struct rb_node *ptr[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
	u32 last_ptr_id[TC_HTB_MAXDEPTH][TC_HTB_NUMPRIO];
L
Linus Torvalds 已提交
144

S
Stephen Hemminger 已提交
145 146
	/* self wait list - roots of wait PQs per row */
	struct rb_root wait_pq[TC_HTB_MAXDEPTH];
L
Linus Torvalds 已提交
147

S
Stephen Hemminger 已提交
148
	/* time of nearest event per level (row) */
149
	psched_time_t near_ev_cache[TC_HTB_MAXDEPTH];
L
Linus Torvalds 已提交
150

S
Stephen Hemminger 已提交
151
	int defcls;		/* class where unclassified flows go to */
L
Linus Torvalds 已提交
152

S
Stephen Hemminger 已提交
153 154
	/* filters for qdisc itself */
	struct tcf_proto *filter_list;
L
Linus Torvalds 已提交
155

S
Stephen Hemminger 已提交
156 157
	int rate2quantum;	/* quant = rate / rate2quantum */
	psched_time_t now;	/* cached dequeue time */
158
	struct qdisc_watchdog watchdog;
L
Linus Torvalds 已提交
159

S
Stephen Hemminger 已提交
160 161 162 163 164
	/* non shaped skbs; let them go directly thru */
	struct sk_buff_head direct_queue;
	int direct_qlen;	/* max qlen of above */

	long direct_pkts;
165 166 167

#define HTB_WARN_TOOMANYEVENTS	0x1
	unsigned int warned;	/* only one warning */
168
	struct work_struct work;
L
Linus Torvalds 已提交
169 170
};

V
Vimalkumar 已提交
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
static u64 l2t_ns(struct htb_rate_cfg *r, unsigned int len)
{
	return ((u64)len * r->mult) >> r->shift;
}

static void htb_precompute_ratedata(struct htb_rate_cfg *r)
{
	u64 factor;
	u64 mult;
	int shift;

	r->shift = 0;
	r->mult = 1;
	/*
	 * Calibrate mult, shift so that token counting is accurate
	 * for smallest packet size (64 bytes).  Token (time in ns) is
	 * computed as (bytes * 8) * NSEC_PER_SEC / rate_bps.  It will
	 * work as long as the smallest packet transfer time can be
	 * accurately represented in nanosec.
	 */
	if (r->rate_bps > 0) {
		/*
		 * Higher shift gives better accuracy.  Find the largest
		 * shift such that mult fits in 32 bits.
		 */
		for (shift = 0; shift < 16; shift++) {
			r->shift = shift;
			factor = 8LLU * NSEC_PER_SEC * (1 << r->shift);
			mult = div64_u64(factor, r->rate_bps);
			if (mult > UINT_MAX)
				break;
		}

		r->shift = shift - 1;
		factor = 8LLU * NSEC_PER_SEC * (1 << r->shift);
		r->mult = div64_u64(factor, r->rate_bps);
	}
}

L
Linus Torvalds 已提交
210
/* find class in global hash table using given handle */
S
Stephen Hemminger 已提交
211
static inline struct htb_class *htb_find(u32 handle, struct Qdisc *sch)
L
Linus Torvalds 已提交
212 213
{
	struct htb_sched *q = qdisc_priv(sch);
214
	struct Qdisc_class_common *clc;
215

216 217
	clc = qdisc_class_find(&q->clhash, handle);
	if (clc == NULL)
L
Linus Torvalds 已提交
218
		return NULL;
219
	return container_of(clc, struct htb_class, common);
L
Linus Torvalds 已提交
220 221 222 223 224 225 226 227 228 229
}

/**
 * htb_classify - classify a packet into class
 *
 * It returns NULL if the packet should be dropped or -1 if the packet
 * should be passed directly thru. In all other cases leaf class is returned.
 * We allow direct class selection by classid in priority. The we examine
 * filters in qdisc and in inner nodes (if higher filter points to the inner
 * node). If we end up with classid MAJOR:0 we enqueue the skb into special
230
 * internal fifo (direct). These packets then go directly thru. If we still
L
Lucas De Marchi 已提交
231
 * have no valid leaf we try to use MAJOR:default leaf. It still unsuccessful
L
Linus Torvalds 已提交
232 233
 * then finish and return direct queue.
 */
E
Eric Dumazet 已提交
234
#define HTB_DIRECT ((struct htb_class *)-1L)
L
Linus Torvalds 已提交
235

S
Stephen Hemminger 已提交
236 237
static struct htb_class *htb_classify(struct sk_buff *skb, struct Qdisc *sch,
				      int *qerr)
L
Linus Torvalds 已提交
238 239 240 241 242 243 244 245
{
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl;
	struct tcf_result res;
	struct tcf_proto *tcf;
	int result;

	/* allow to select class by setting skb->priority to valid classid;
E
Eric Dumazet 已提交
246 247 248
	 * note that nfmark can be used too by attaching filter fw with no
	 * rules in it
	 */
L
Linus Torvalds 已提交
249
	if (skb->priority == sch->handle)
S
Stephen Hemminger 已提交
250
		return HTB_DIRECT;	/* X:0 (direct flow) selected */
E
Eric Dumazet 已提交
251 252
	cl = htb_find(skb->priority, sch);
	if (cl && cl->level == 0)
L
Linus Torvalds 已提交
253 254
		return cl;

255
	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
L
Linus Torvalds 已提交
256 257 258 259 260
	tcf = q->filter_list;
	while (tcf && (result = tc_classify(skb, tcf, &res)) >= 0) {
#ifdef CONFIG_NET_CLS_ACT
		switch (result) {
		case TC_ACT_QUEUED:
S
Stephen Hemminger 已提交
261
		case TC_ACT_STOLEN:
262
			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
L
Linus Torvalds 已提交
263 264 265 266
		case TC_ACT_SHOT:
			return NULL;
		}
#endif
E
Eric Dumazet 已提交
267 268
		cl = (void *)res.class;
		if (!cl) {
L
Linus Torvalds 已提交
269
			if (res.classid == sch->handle)
S
Stephen Hemminger 已提交
270
				return HTB_DIRECT;	/* X:0 (direct flow) */
E
Eric Dumazet 已提交
271 272
			cl = htb_find(res.classid, sch);
			if (!cl)
S
Stephen Hemminger 已提交
273
				break;	/* filter selected invalid classid */
L
Linus Torvalds 已提交
274 275
		}
		if (!cl->level)
S
Stephen Hemminger 已提交
276
			return cl;	/* we hit leaf; return it */
L
Linus Torvalds 已提交
277 278 279 280 281

		/* we have got inner class; apply inner filter chain */
		tcf = cl->filter_list;
	}
	/* classification failed; try to use default class */
S
Stephen Hemminger 已提交
282
	cl = htb_find(TC_H_MAKE(TC_H_MAJ(sch->handle), q->defcls), sch);
L
Linus Torvalds 已提交
283
	if (!cl || cl->level)
S
Stephen Hemminger 已提交
284
		return HTB_DIRECT;	/* bad default .. this is safe bet */
L
Linus Torvalds 已提交
285 286 287 288 289 290 291 292 293
	return cl;
}

/**
 * htb_add_to_id_tree - adds class to the round robin list
 *
 * Routine adds class to the list (actually tree) sorted by classid.
 * Make sure that class is not already on such list for given prio.
 */
S
Stephen Hemminger 已提交
294 295
static void htb_add_to_id_tree(struct rb_root *root,
			       struct htb_class *cl, int prio)
L
Linus Torvalds 已提交
296 297
{
	struct rb_node **p = &root->rb_node, *parent = NULL;
298

L
Linus Torvalds 已提交
299
	while (*p) {
S
Stephen Hemminger 已提交
300 301
		struct htb_class *c;
		parent = *p;
L
Linus Torvalds 已提交
302
		c = rb_entry(parent, struct htb_class, node[prio]);
303

304
		if (cl->common.classid > c->common.classid)
L
Linus Torvalds 已提交
305
			p = &parent->rb_right;
S
Stephen Hemminger 已提交
306
		else
L
Linus Torvalds 已提交
307 308 309 310 311 312 313 314 315 316 317 318 319
			p = &parent->rb_left;
	}
	rb_link_node(&cl->node[prio], parent, p);
	rb_insert_color(&cl->node[prio], root);
}

/**
 * htb_add_to_wait_tree - adds class to the event queue with delay
 *
 * The class is added to priority event queue to indicate that class will
 * change its mode in cl->pq_key microseconds. Make sure that class is not
 * already in the queue.
 */
S
Stephen Hemminger 已提交
320
static void htb_add_to_wait_tree(struct htb_sched *q,
V
Vimalkumar 已提交
321
				 struct htb_class *cl, s64 delay)
L
Linus Torvalds 已提交
322 323
{
	struct rb_node **p = &q->wait_pq[cl->level].rb_node, *parent = NULL;
324

325 326
	cl->pq_key = q->now + delay;
	if (cl->pq_key == q->now)
L
Linus Torvalds 已提交
327 328 329
		cl->pq_key++;

	/* update the nearest event cache */
330
	if (q->near_ev_cache[cl->level] > cl->pq_key)
L
Linus Torvalds 已提交
331
		q->near_ev_cache[cl->level] = cl->pq_key;
S
Stephen Hemminger 已提交
332

L
Linus Torvalds 已提交
333
	while (*p) {
S
Stephen Hemminger 已提交
334 335
		struct htb_class *c;
		parent = *p;
L
Linus Torvalds 已提交
336
		c = rb_entry(parent, struct htb_class, pq_node);
337
		if (cl->pq_key >= c->pq_key)
L
Linus Torvalds 已提交
338
			p = &parent->rb_right;
S
Stephen Hemminger 已提交
339
		else
L
Linus Torvalds 已提交
340 341 342 343 344 345 346 347 348 349 350 351
			p = &parent->rb_left;
	}
	rb_link_node(&cl->pq_node, parent, p);
	rb_insert_color(&cl->pq_node, &q->wait_pq[cl->level]);
}

/**
 * htb_next_rb_node - finds next node in binary tree
 *
 * When we are past last key we return NULL.
 * Average complexity is 2 steps per call.
 */
S
Stephen Hemminger 已提交
352
static inline void htb_next_rb_node(struct rb_node **n)
L
Linus Torvalds 已提交
353 354 355 356 357 358 359 360 361 362
{
	*n = rb_next(*n);
}

/**
 * htb_add_class_to_row - add class to its row
 *
 * The class is added to row at priorities marked in mask.
 * It does nothing if mask == 0.
 */
S
Stephen Hemminger 已提交
363 364
static inline void htb_add_class_to_row(struct htb_sched *q,
					struct htb_class *cl, int mask)
L
Linus Torvalds 已提交
365 366 367 368 369
{
	q->row_mask[cl->level] |= mask;
	while (mask) {
		int prio = ffz(~mask);
		mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
370
		htb_add_to_id_tree(q->row[cl->level] + prio, cl, prio);
L
Linus Torvalds 已提交
371 372 373
	}
}

S
Stephen Hemminger 已提交
374 375 376
/* If this triggers, it is a bug in this code, but it need not be fatal */
static void htb_safe_rb_erase(struct rb_node *rb, struct rb_root *root)
{
377
	if (RB_EMPTY_NODE(rb)) {
S
Stephen Hemminger 已提交
378 379 380 381 382 383 384 385
		WARN_ON(1);
	} else {
		rb_erase(rb, root);
		RB_CLEAR_NODE(rb);
	}
}


L
Linus Torvalds 已提交
386 387 388 389 390 391
/**
 * htb_remove_class_from_row - removes class from its row
 *
 * The class is removed from row at priorities marked in mask.
 * It does nothing if mask == 0.
 */
S
Stephen Hemminger 已提交
392 393
static inline void htb_remove_class_from_row(struct htb_sched *q,
						 struct htb_class *cl, int mask)
L
Linus Torvalds 已提交
394 395
{
	int m = 0;
396

L
Linus Torvalds 已提交
397 398
	while (mask) {
		int prio = ffz(~mask);
S
Stephen Hemminger 已提交
399

L
Linus Torvalds 已提交
400
		mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
401 402
		if (q->ptr[cl->level][prio] == cl->node + prio)
			htb_next_rb_node(q->ptr[cl->level] + prio);
S
Stephen Hemminger 已提交
403 404

		htb_safe_rb_erase(cl->node + prio, q->row[cl->level] + prio);
S
Stephen Hemminger 已提交
405
		if (!q->row[cl->level][prio].rb_node)
L
Linus Torvalds 已提交
406 407 408 409 410 411 412 413 414
			m |= 1 << prio;
	}
	q->row_mask[cl->level] &= ~m;
}

/**
 * htb_activate_prios - creates active classe's feed chain
 *
 * The class is connected to ancestors and/or appropriate rows
415
 * for priorities it is participating on. cl->cmode must be new
L
Linus Torvalds 已提交
416 417
 * (activated) mode. It does nothing if cl->prio_activity == 0.
 */
S
Stephen Hemminger 已提交
418
static void htb_activate_prios(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
419 420
{
	struct htb_class *p = cl->parent;
S
Stephen Hemminger 已提交
421
	long m, mask = cl->prio_activity;
L
Linus Torvalds 已提交
422 423

	while (cl->cmode == HTB_MAY_BORROW && p && mask) {
S
Stephen Hemminger 已提交
424 425
		m = mask;
		while (m) {
L
Linus Torvalds 已提交
426 427
			int prio = ffz(~m);
			m &= ~(1 << prio);
S
Stephen Hemminger 已提交
428

L
Linus Torvalds 已提交
429 430
			if (p->un.inner.feed[prio].rb_node)
				/* parent already has its feed in use so that
E
Eric Dumazet 已提交
431 432
				 * reset bit in mask as parent is already ok
				 */
L
Linus Torvalds 已提交
433
				mask &= ~(1 << prio);
S
Stephen Hemminger 已提交
434 435

			htb_add_to_id_tree(p->un.inner.feed + prio, cl, prio);
L
Linus Torvalds 已提交
436 437
		}
		p->prio_activity |= mask;
S
Stephen Hemminger 已提交
438 439
		cl = p;
		p = cl->parent;
440

L
Linus Torvalds 已提交
441 442
	}
	if (cl->cmode == HTB_CAN_SEND && mask)
S
Stephen Hemminger 已提交
443
		htb_add_class_to_row(q, cl, mask);
L
Linus Torvalds 已提交
444 445 446 447 448
}

/**
 * htb_deactivate_prios - remove class from feed chain
 *
449
 * cl->cmode must represent old mode (before deactivation). It does
L
Linus Torvalds 已提交
450 451 452 453 454 455
 * nothing if cl->prio_activity == 0. Class is removed from all feed
 * chains and rows.
 */
static void htb_deactivate_prios(struct htb_sched *q, struct htb_class *cl)
{
	struct htb_class *p = cl->parent;
S
Stephen Hemminger 已提交
456
	long m, mask = cl->prio_activity;
L
Linus Torvalds 已提交
457 458

	while (cl->cmode == HTB_MAY_BORROW && p && mask) {
S
Stephen Hemminger 已提交
459 460
		m = mask;
		mask = 0;
L
Linus Torvalds 已提交
461 462 463
		while (m) {
			int prio = ffz(~m);
			m &= ~(1 << prio);
S
Stephen Hemminger 已提交
464 465

			if (p->un.inner.ptr[prio] == cl->node + prio) {
L
Linus Torvalds 已提交
466
				/* we are removing child which is pointed to from
E
Eric Dumazet 已提交
467 468 469
				 * parent feed - forget the pointer but remember
				 * classid
				 */
470
				p->un.inner.last_ptr_id[prio] = cl->common.classid;
L
Linus Torvalds 已提交
471 472
				p->un.inner.ptr[prio] = NULL;
			}
S
Stephen Hemminger 已提交
473

S
Stephen Hemminger 已提交
474
			htb_safe_rb_erase(cl->node + prio, p->un.inner.feed + prio);
S
Stephen Hemminger 已提交
475 476

			if (!p->un.inner.feed[prio].rb_node)
L
Linus Torvalds 已提交
477 478
				mask |= 1 << prio;
		}
479

L
Linus Torvalds 已提交
480
		p->prio_activity &= ~mask;
S
Stephen Hemminger 已提交
481 482
		cl = p;
		p = cl->parent;
483

L
Linus Torvalds 已提交
484
	}
S
Stephen Hemminger 已提交
485 486
	if (cl->cmode == HTB_CAN_SEND && mask)
		htb_remove_class_from_row(q, cl, mask);
L
Linus Torvalds 已提交
487 488
}

V
Vimalkumar 已提交
489
static inline s64 htb_lowater(const struct htb_class *cl)
490
{
491 492 493 494
	if (htb_hysteresis)
		return cl->cmode != HTB_CANT_SEND ? -cl->cbuffer : 0;
	else
		return 0;
495
}
V
Vimalkumar 已提交
496
static inline s64 htb_hiwater(const struct htb_class *cl)
497
{
498 499 500 501
	if (htb_hysteresis)
		return cl->cmode == HTB_CAN_SEND ? -cl->buffer : 0;
	else
		return 0;
502
}
503

504

L
Linus Torvalds 已提交
505 506 507 508 509
/**
 * htb_class_mode - computes and returns current class mode
 *
 * It computes cl's mode at time cl->t_c+diff and returns it. If mode
 * is not HTB_CAN_SEND then cl->pq_key is updated to time difference
510
 * from now to time when cl will change its state.
L
Linus Torvalds 已提交
511
 * Also it is worth to note that class mode doesn't change simply
512
 * at cl->{c,}tokens == 0 but there can rather be hysteresis of
L
Linus Torvalds 已提交
513 514 515
 * 0 .. -cl->{c,}buffer range. It is meant to limit number of
 * mode transitions per time unit. The speed gain is about 1/6.
 */
S
Stephen Hemminger 已提交
516
static inline enum htb_cmode
V
Vimalkumar 已提交
517
htb_class_mode(struct htb_class *cl, s64 *diff)
L
Linus Torvalds 已提交
518
{
V
Vimalkumar 已提交
519
	s64 toks;
L
Linus Torvalds 已提交
520

S
Stephen Hemminger 已提交
521 522 523 524
	if ((toks = (cl->ctokens + *diff)) < htb_lowater(cl)) {
		*diff = -toks;
		return HTB_CANT_SEND;
	}
525

S
Stephen Hemminger 已提交
526 527
	if ((toks = (cl->tokens + *diff)) >= htb_hiwater(cl))
		return HTB_CAN_SEND;
L
Linus Torvalds 已提交
528

S
Stephen Hemminger 已提交
529 530
	*diff = -toks;
	return HTB_MAY_BORROW;
L
Linus Torvalds 已提交
531 532 533 534 535 536 537 538 539 540 541
}

/**
 * htb_change_class_mode - changes classe's mode
 *
 * This should be the only way how to change classe's mode under normal
 * cirsumstances. Routine will update feed lists linkage, change mode
 * and add class to the wait event queue if appropriate. New mode should
 * be different from old one and cl->pq_key has to be valid if changing
 * to mode other than HTB_CAN_SEND (see htb_add_to_wait_tree).
 */
S
Stephen Hemminger 已提交
542
static void
V
Vimalkumar 已提交
543
htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, s64 *diff)
S
Stephen Hemminger 已提交
544 545
{
	enum htb_cmode new_mode = htb_class_mode(cl, diff);
L
Linus Torvalds 已提交
546 547

	if (new_mode == cl->cmode)
S
Stephen Hemminger 已提交
548 549 550 551 552
		return;

	if (cl->prio_activity) {	/* not necessary: speed optimization */
		if (cl->cmode != HTB_CANT_SEND)
			htb_deactivate_prios(q, cl);
L
Linus Torvalds 已提交
553
		cl->cmode = new_mode;
S
Stephen Hemminger 已提交
554 555 556
		if (new_mode != HTB_CANT_SEND)
			htb_activate_prios(q, cl);
	} else
L
Linus Torvalds 已提交
557 558 559 560
		cl->cmode = new_mode;
}

/**
561
 * htb_activate - inserts leaf cl into appropriate active feeds
L
Linus Torvalds 已提交
562 563 564 565 566
 *
 * Routine learns (new) priority of leaf and activates feed chain
 * for the prio. It can be called on already active leaf safely.
 * It also adds leaf into droplist.
 */
S
Stephen Hemminger 已提交
567
static inline void htb_activate(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
568
{
569
	WARN_ON(cl->level || !cl->un.leaf.q || !cl->un.leaf.q->q.qlen);
570

L
Linus Torvalds 已提交
571
	if (!cl->prio_activity) {
572
		cl->prio_activity = 1 << cl->prio;
S
Stephen Hemminger 已提交
573 574
		htb_activate_prios(q, cl);
		list_add_tail(&cl->un.leaf.drop_list,
575
			      q->drops + cl->prio);
L
Linus Torvalds 已提交
576 577 578 579
	}
}

/**
580
 * htb_deactivate - remove leaf cl from active feeds
L
Linus Torvalds 已提交
581 582 583 584
 *
 * Make sure that leaf is active. In the other words it can't be called
 * with non-active leaf. It also removes class from the drop list.
 */
S
Stephen Hemminger 已提交
585
static inline void htb_deactivate(struct htb_sched *q, struct htb_class *cl)
L
Linus Torvalds 已提交
586
{
587
	WARN_ON(!cl->prio_activity);
588

S
Stephen Hemminger 已提交
589
	htb_deactivate_prios(q, cl);
L
Linus Torvalds 已提交
590 591 592 593 594 595
	cl->prio_activity = 0;
	list_del_init(&cl->un.leaf.drop_list);
}

static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch)
{
596
	int uninitialized_var(ret);
S
Stephen Hemminger 已提交
597 598 599 600 601 602 603 604 605
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl = htb_classify(skb, sch, &ret);

	if (cl == HTB_DIRECT) {
		/* enqueue to helper queue */
		if (q->direct_queue.qlen < q->direct_qlen) {
			__skb_queue_tail(&q->direct_queue, skb);
			q->direct_pkts++;
		} else {
606
			return qdisc_drop(skb, sch);
S
Stephen Hemminger 已提交
607
		}
L
Linus Torvalds 已提交
608
#ifdef CONFIG_NET_CLS_ACT
S
Stephen Hemminger 已提交
609
	} else if (!cl) {
610
		if (ret & __NET_XMIT_BYPASS)
S
Stephen Hemminger 已提交
611 612 613
			sch->qstats.drops++;
		kfree_skb(skb);
		return ret;
L
Linus Torvalds 已提交
614
#endif
615 616 617 618 619
	} else if ((ret = qdisc_enqueue(skb, cl->un.leaf.q)) != NET_XMIT_SUCCESS) {
		if (net_xmit_drop_count(ret)) {
			sch->qstats.drops++;
			cl->qstats.drops++;
		}
620
		return ret;
S
Stephen Hemminger 已提交
621 622 623 624 625 626
	} else {
		htb_activate(q, cl);
	}

	sch->q.qlen++;
	return NET_XMIT_SUCCESS;
L
Linus Torvalds 已提交
627 628
}

V
Vimalkumar 已提交
629
static inline void htb_accnt_tokens(struct htb_class *cl, int bytes, s64 diff)
630
{
V
Vimalkumar 已提交
631
	s64 toks = diff + cl->tokens;
632 633 634

	if (toks > cl->buffer)
		toks = cl->buffer;
V
Vimalkumar 已提交
635
	toks -= (s64) l2t_ns(&cl->rate, bytes);
636 637 638 639 640 641
	if (toks <= -cl->mbuffer)
		toks = 1 - cl->mbuffer;

	cl->tokens = toks;
}

V
Vimalkumar 已提交
642
static inline void htb_accnt_ctokens(struct htb_class *cl, int bytes, s64 diff)
643
{
V
Vimalkumar 已提交
644
	s64 toks = diff + cl->ctokens;
645 646 647

	if (toks > cl->cbuffer)
		toks = cl->cbuffer;
V
Vimalkumar 已提交
648
	toks -= (s64) l2t_ns(&cl->ceil, bytes);
649 650 651 652 653 654
	if (toks <= -cl->mbuffer)
		toks = 1 - cl->mbuffer;

	cl->ctokens = toks;
}

L
Linus Torvalds 已提交
655 656 657 658 659 660 661 662 663 664 665
/**
 * htb_charge_class - charges amount "bytes" to leaf and ancestors
 *
 * Routine assumes that packet "bytes" long was dequeued from leaf cl
 * borrowing from "level". It accounts bytes to ceil leaky bucket for
 * leaf and all ancestors and to rate bucket for ancestors at levels
 * "level" and higher. It also handles possible change of mode resulting
 * from the update. Note that mode can also increase here (MAY_BORROW to
 * CAN_SEND) because we can use more precise clock that event queue here.
 * In such case we remove class from event queue first.
 */
S
Stephen Hemminger 已提交
666
static void htb_charge_class(struct htb_sched *q, struct htb_class *cl,
667
			     int level, struct sk_buff *skb)
S
Stephen Hemminger 已提交
668
{
669
	int bytes = qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
670
	enum htb_cmode old_mode;
V
Vimalkumar 已提交
671
	s64 diff;
L
Linus Torvalds 已提交
672 673

	while (cl) {
V
Vimalkumar 已提交
674
		diff = min_t(s64, q->now - cl->t_c, cl->mbuffer);
L
Linus Torvalds 已提交
675
		if (cl->level >= level) {
S
Stephen Hemminger 已提交
676 677
			if (cl->level == level)
				cl->xstats.lends++;
678
			htb_accnt_tokens(cl, bytes, diff);
L
Linus Torvalds 已提交
679 680
		} else {
			cl->xstats.borrows++;
S
Stephen Hemminger 已提交
681
			cl->tokens += diff;	/* we moved t_c; update tokens */
L
Linus Torvalds 已提交
682
		}
683
		htb_accnt_ctokens(cl, bytes, diff);
L
Linus Torvalds 已提交
684 685
		cl->t_c = q->now;

S
Stephen Hemminger 已提交
686 687 688
		old_mode = cl->cmode;
		diff = 0;
		htb_change_class_mode(q, cl, &diff);
L
Linus Torvalds 已提交
689 690
		if (old_mode != cl->cmode) {
			if (old_mode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
691
				htb_safe_rb_erase(&cl->pq_node, q->wait_pq + cl->level);
L
Linus Torvalds 已提交
692
			if (cl->cmode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
693
				htb_add_to_wait_tree(q, cl, diff);
L
Linus Torvalds 已提交
694 695
		}

696 697 698 699
		/* update basic stats except for leaves which are already updated */
		if (cl->level)
			bstats_update(&cl->bstats, skb);

L
Linus Torvalds 已提交
700 701 702 703 704 705 706
		cl = cl->parent;
	}
}

/**
 * htb_do_events - make mode changes to classes at the level
 *
707
 * Scans event queue for pending events and applies them. Returns time of
708
 * next pending event (0 for no event in pq, q->now for too many events).
709
 * Note: Applied are events whose have cl->pq_key <= q->now.
L
Linus Torvalds 已提交
710
 */
711 712
static psched_time_t htb_do_events(struct htb_sched *q, int level,
				   unsigned long start)
L
Linus Torvalds 已提交
713
{
714
	/* don't run for longer than 2 jiffies; 2 is used instead of
E
Eric Dumazet 已提交
715 716 717
	 * 1 to simplify things when jiffy is going to be incremented
	 * too soon
	 */
718
	unsigned long stop_at = start + 2;
719
	while (time_before(jiffies, stop_at)) {
L
Linus Torvalds 已提交
720
		struct htb_class *cl;
V
Vimalkumar 已提交
721
		s64 diff;
722 723
		struct rb_node *p = rb_first(&q->wait_pq[level]);

S
Stephen Hemminger 已提交
724 725
		if (!p)
			return 0;
L
Linus Torvalds 已提交
726 727

		cl = rb_entry(p, struct htb_class, pq_node);
728 729 730
		if (cl->pq_key > q->now)
			return cl->pq_key;

S
Stephen Hemminger 已提交
731
		htb_safe_rb_erase(p, q->wait_pq + level);
V
Vimalkumar 已提交
732
		diff = min_t(s64, q->now - cl->t_c, cl->mbuffer);
S
Stephen Hemminger 已提交
733
		htb_change_class_mode(q, cl, &diff);
L
Linus Torvalds 已提交
734
		if (cl->cmode != HTB_CAN_SEND)
S
Stephen Hemminger 已提交
735
			htb_add_to_wait_tree(q, cl, diff);
L
Linus Torvalds 已提交
736
	}
737 738

	/* too much load - let's continue after a break for scheduling */
739
	if (!(q->warned & HTB_WARN_TOOMANYEVENTS)) {
E
Eric Dumazet 已提交
740
		pr_warning("htb: too many events!\n");
741 742
		q->warned |= HTB_WARN_TOOMANYEVENTS;
	}
743 744

	return q->now;
L
Linus Torvalds 已提交
745 746 747
}

/* Returns class->node+prio from id-tree where classe's id is >= id. NULL
E
Eric Dumazet 已提交
748 749
 * is no such one exists.
 */
S
Stephen Hemminger 已提交
750 751
static struct rb_node *htb_id_find_next_upper(int prio, struct rb_node *n,
					      u32 id)
L
Linus Torvalds 已提交
752 753 754
{
	struct rb_node *r = NULL;
	while (n) {
S
Stephen Hemminger 已提交
755 756 757
		struct htb_class *cl =
		    rb_entry(n, struct htb_class, node[prio]);

758
		if (id > cl->common.classid) {
L
Linus Torvalds 已提交
759
			n = n->rb_right;
760
		} else if (id < cl->common.classid) {
L
Linus Torvalds 已提交
761 762
			r = n;
			n = n->rb_left;
763 764
		} else {
			return n;
L
Linus Torvalds 已提交
765 766 767 768 769 770 771 772 773 774
		}
	}
	return r;
}

/**
 * htb_lookup_leaf - returns next leaf class in DRR order
 *
 * Find leaf where current feed pointers points to.
 */
S
Stephen Hemminger 已提交
775 776
static struct htb_class *htb_lookup_leaf(struct rb_root *tree, int prio,
					 struct rb_node **pptr, u32 * pid)
L
Linus Torvalds 已提交
777 778 779 780 781 782
{
	int i;
	struct {
		struct rb_node *root;
		struct rb_node **pptr;
		u32 *pid;
S
Stephen Hemminger 已提交
783 784
	} stk[TC_HTB_MAXDEPTH], *sp = stk;

785
	BUG_ON(!tree->rb_node);
L
Linus Torvalds 已提交
786 787 788 789 790
	sp->root = tree->rb_node;
	sp->pptr = pptr;
	sp->pid = pid;

	for (i = 0; i < 65535; i++) {
S
Stephen Hemminger 已提交
791
		if (!*sp->pptr && *sp->pid) {
792
			/* ptr was invalidated but id is valid - try to recover
E
Eric Dumazet 已提交
793 794
			 * the original or next ptr
			 */
S
Stephen Hemminger 已提交
795 796
			*sp->pptr =
			    htb_id_find_next_upper(prio, sp->root, *sp->pid);
L
Linus Torvalds 已提交
797
		}
S
Stephen Hemminger 已提交
798
		*sp->pid = 0;	/* ptr is valid now so that remove this hint as it
E
Eric Dumazet 已提交
799 800
				 * can become out of date quickly
				 */
S
Stephen Hemminger 已提交
801
		if (!*sp->pptr) {	/* we are at right end; rewind & go up */
L
Linus Torvalds 已提交
802
			*sp->pptr = sp->root;
S
Stephen Hemminger 已提交
803
			while ((*sp->pptr)->rb_left)
L
Linus Torvalds 已提交
804 805 806
				*sp->pptr = (*sp->pptr)->rb_left;
			if (sp > stk) {
				sp--;
807 808
				if (!*sp->pptr) {
					WARN_ON(1);
S
Stephen Hemminger 已提交
809
					return NULL;
810
				}
S
Stephen Hemminger 已提交
811
				htb_next_rb_node(sp->pptr);
L
Linus Torvalds 已提交
812 813 814
			}
		} else {
			struct htb_class *cl;
S
Stephen Hemminger 已提交
815 816
			cl = rb_entry(*sp->pptr, struct htb_class, node[prio]);
			if (!cl->level)
L
Linus Torvalds 已提交
817 818
				return cl;
			(++sp)->root = cl->un.inner.feed[prio].rb_node;
S
Stephen Hemminger 已提交
819 820
			sp->pptr = cl->un.inner.ptr + prio;
			sp->pid = cl->un.inner.last_ptr_id + prio;
L
Linus Torvalds 已提交
821 822
		}
	}
823
	WARN_ON(1);
L
Linus Torvalds 已提交
824 825 826 827
	return NULL;
}

/* dequeues packet at given priority and level; call only if
E
Eric Dumazet 已提交
828 829
 * you are sure that there is active class at prio/level
 */
S
Stephen Hemminger 已提交
830 831
static struct sk_buff *htb_dequeue_tree(struct htb_sched *q, int prio,
					int level)
L
Linus Torvalds 已提交
832 833
{
	struct sk_buff *skb = NULL;
S
Stephen Hemminger 已提交
834
	struct htb_class *cl, *start;
L
Linus Torvalds 已提交
835
	/* look initial class up in the row */
S
Stephen Hemminger 已提交
836 837 838 839
	start = cl = htb_lookup_leaf(q->row[level] + prio, prio,
				     q->ptr[level] + prio,
				     q->last_ptr_id[level] + prio);

L
Linus Torvalds 已提交
840 841
	do {
next:
842
		if (unlikely(!cl))
S
Stephen Hemminger 已提交
843
			return NULL;
L
Linus Torvalds 已提交
844 845

		/* class can be empty - it is unlikely but can be true if leaf
E
Eric Dumazet 已提交
846 847 848 849
		 * qdisc drops packets in enqueue routine or if someone used
		 * graft operation on the leaf since last dequeue;
		 * simply deactivate and skip such class
		 */
L
Linus Torvalds 已提交
850 851
		if (unlikely(cl->un.leaf.q->q.qlen == 0)) {
			struct htb_class *next;
S
Stephen Hemminger 已提交
852
			htb_deactivate(q, cl);
L
Linus Torvalds 已提交
853 854 855

			/* row/level might become empty */
			if ((q->row_mask[level] & (1 << prio)) == 0)
S
Stephen Hemminger 已提交
856
				return NULL;
L
Linus Torvalds 已提交
857

S
Stephen Hemminger 已提交
858 859 860 861 862
			next = htb_lookup_leaf(q->row[level] + prio,
					       prio, q->ptr[level] + prio,
					       q->last_ptr_id[level] + prio);

			if (cl == start)	/* fix start if we just deleted it */
L
Linus Torvalds 已提交
863 864 865 866
				start = next;
			cl = next;
			goto next;
		}
S
Stephen Hemminger 已提交
867 868 869

		skb = cl->un.leaf.q->dequeue(cl->un.leaf.q);
		if (likely(skb != NULL))
L
Linus Torvalds 已提交
870
			break;
871

872
		qdisc_warn_nonwc("htb", cl->un.leaf.q);
S
Stephen Hemminger 已提交
873 874 875 876 877
		htb_next_rb_node((level ? cl->parent->un.inner.ptr : q->
				  ptr[0]) + prio);
		cl = htb_lookup_leaf(q->row[level] + prio, prio,
				     q->ptr[level] + prio,
				     q->last_ptr_id[level] + prio);
L
Linus Torvalds 已提交
878 879 880 881

	} while (cl != start);

	if (likely(skb != NULL)) {
E
Eric Dumazet 已提交
882
		bstats_update(&cl->bstats, skb);
883 884
		cl->un.leaf.deficit[level] -= qdisc_pkt_len(skb);
		if (cl->un.leaf.deficit[level] < 0) {
885
			cl->un.leaf.deficit[level] += cl->quantum;
S
Stephen Hemminger 已提交
886 887
			htb_next_rb_node((level ? cl->parent->un.inner.ptr : q->
					  ptr[0]) + prio);
L
Linus Torvalds 已提交
888 889
		}
		/* this used to be after charge_class but this constelation
E
Eric Dumazet 已提交
890 891
		 * gives us slightly better performance
		 */
L
Linus Torvalds 已提交
892
		if (!cl->un.leaf.q->q.qlen)
S
Stephen Hemminger 已提交
893
			htb_deactivate(q, cl);
894
		htb_charge_class(q, cl, level, skb);
L
Linus Torvalds 已提交
895 896 897 898 899 900
	}
	return skb;
}

static struct sk_buff *htb_dequeue(struct Qdisc *sch)
{
901
	struct sk_buff *skb;
L
Linus Torvalds 已提交
902 903
	struct htb_sched *q = qdisc_priv(sch);
	int level;
904
	psched_time_t next_event;
905
	unsigned long start_at;
L
Linus Torvalds 已提交
906 907

	/* try to dequeue direct packets as high prio (!) to minimize cpu work */
S
Stephen Hemminger 已提交
908 909
	skb = __skb_dequeue(&q->direct_queue);
	if (skb != NULL) {
910 911
ok:
		qdisc_bstats_update(sch, skb);
912
		qdisc_unthrottled(sch);
L
Linus Torvalds 已提交
913 914 915 916
		sch->q.qlen--;
		return skb;
	}

S
Stephen Hemminger 已提交
917 918
	if (!sch->q.qlen)
		goto fin;
V
Vimalkumar 已提交
919
	q->now = ktime_to_ns(ktime_get());
920
	start_at = jiffies;
L
Linus Torvalds 已提交
921

S
Stefan Hasko 已提交
922
	next_event = q->now + 5LLU * NSEC_PER_SEC;
923

L
Linus Torvalds 已提交
924 925 926
	for (level = 0; level < TC_HTB_MAXDEPTH; level++) {
		/* common case optimization - skip event handler quickly */
		int m;
927 928 929
		psched_time_t event;

		if (q->now >= q->near_ev_cache[level]) {
930
			event = htb_do_events(q, level, start_at);
931
			if (!event)
V
Vimalkumar 已提交
932
				event = q->now + NSEC_PER_SEC;
933
			q->near_ev_cache[level] = event;
L
Linus Torvalds 已提交
934
		} else
935 936
			event = q->near_ev_cache[level];

937
		if (next_event > event)
938
			next_event = event;
S
Stephen Hemminger 已提交
939

L
Linus Torvalds 已提交
940 941
		m = ~q->row_mask[level];
		while (m != (int)(-1)) {
S
Stephen Hemminger 已提交
942
			int prio = ffz(m);
E
Eric Dumazet 已提交
943

L
Linus Torvalds 已提交
944
			m |= 1 << prio;
S
Stephen Hemminger 已提交
945
			skb = htb_dequeue_tree(q, prio, level);
946 947
			if (likely(skb != NULL))
				goto ok;
L
Linus Torvalds 已提交
948 949
		}
	}
950
	sch->qstats.overlimits++;
V
Vimalkumar 已提交
951 952 953 954 955 956 957 958 959
	if (likely(next_event > q->now)) {
		if (!test_bit(__QDISC_STATE_DEACTIVATED,
			      &qdisc_root_sleeping(q->watchdog.qdisc)->state)) {
			ktime_t time = ns_to_ktime(next_event);
			qdisc_throttled(q->watchdog.qdisc);
			hrtimer_start(&q->watchdog.timer, time,
				      HRTIMER_MODE_ABS);
		}
	} else {
960
		schedule_work(&q->work);
V
Vimalkumar 已提交
961
	}
L
Linus Torvalds 已提交
962 963 964 965 966
fin:
	return skb;
}

/* try to drop from each class (by prio) until one succeed */
S
Stephen Hemminger 已提交
967
static unsigned int htb_drop(struct Qdisc *sch)
L
Linus Torvalds 已提交
968 969 970 971 972 973
{
	struct htb_sched *q = qdisc_priv(sch);
	int prio;

	for (prio = TC_HTB_NUMPRIO - 1; prio >= 0; prio--) {
		struct list_head *p;
S
Stephen Hemminger 已提交
974
		list_for_each(p, q->drops + prio) {
L
Linus Torvalds 已提交
975 976 977
			struct htb_class *cl = list_entry(p, struct htb_class,
							  un.leaf.drop_list);
			unsigned int len;
S
Stephen Hemminger 已提交
978 979
			if (cl->un.leaf.q->ops->drop &&
			    (len = cl->un.leaf.q->ops->drop(cl->un.leaf.q))) {
L
Linus Torvalds 已提交
980 981
				sch->q.qlen--;
				if (!cl->un.leaf.q->q.qlen)
S
Stephen Hemminger 已提交
982
					htb_deactivate(q, cl);
L
Linus Torvalds 已提交
983 984 985 986 987 988 989 990 991
				return len;
			}
		}
	}
	return 0;
}

/* reset all classes */
/* always caled under BH & queue lock */
S
Stephen Hemminger 已提交
992
static void htb_reset(struct Qdisc *sch)
L
Linus Torvalds 已提交
993 994
{
	struct htb_sched *q = qdisc_priv(sch);
995 996 997
	struct htb_class *cl;
	struct hlist_node *n;
	unsigned int i;
998

999 1000
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode) {
L
Linus Torvalds 已提交
1001
			if (cl->level)
S
Stephen Hemminger 已提交
1002
				memset(&cl->un.inner, 0, sizeof(cl->un.inner));
L
Linus Torvalds 已提交
1003
			else {
S
Stephen Hemminger 已提交
1004
				if (cl->un.leaf.q)
L
Linus Torvalds 已提交
1005 1006 1007 1008 1009 1010 1011 1012
					qdisc_reset(cl->un.leaf.q);
				INIT_LIST_HEAD(&cl->un.leaf.drop_list);
			}
			cl->prio_activity = 0;
			cl->cmode = HTB_CAN_SEND;

		}
	}
1013
	qdisc_watchdog_cancel(&q->watchdog);
L
Linus Torvalds 已提交
1014 1015
	__skb_queue_purge(&q->direct_queue);
	sch->q.qlen = 0;
S
Stephen Hemminger 已提交
1016 1017 1018 1019
	memset(q->row, 0, sizeof(q->row));
	memset(q->row_mask, 0, sizeof(q->row_mask));
	memset(q->wait_pq, 0, sizeof(q->wait_pq));
	memset(q->ptr, 0, sizeof(q->ptr));
L
Linus Torvalds 已提交
1020
	for (i = 0; i < TC_HTB_NUMPRIO; i++)
S
Stephen Hemminger 已提交
1021
		INIT_LIST_HEAD(q->drops + i);
L
Linus Torvalds 已提交
1022 1023
}

1024 1025 1026 1027 1028 1029 1030
static const struct nla_policy htb_policy[TCA_HTB_MAX + 1] = {
	[TCA_HTB_PARMS]	= { .len = sizeof(struct tc_htb_opt) },
	[TCA_HTB_INIT]	= { .len = sizeof(struct tc_htb_glob) },
	[TCA_HTB_CTAB]	= { .type = NLA_BINARY, .len = TC_RTAB_SIZE },
	[TCA_HTB_RTAB]	= { .type = NLA_BINARY, .len = TC_RTAB_SIZE },
};

1031 1032 1033 1034 1035 1036 1037 1038
static void htb_work_func(struct work_struct *work)
{
	struct htb_sched *q = container_of(work, struct htb_sched, work);
	struct Qdisc *sch = q->watchdog.qdisc;

	__netif_schedule(qdisc_root(sch));
}

1039
static int htb_init(struct Qdisc *sch, struct nlattr *opt)
L
Linus Torvalds 已提交
1040 1041
{
	struct htb_sched *q = qdisc_priv(sch);
1042
	struct nlattr *tb[TCA_HTB_INIT + 1];
L
Linus Torvalds 已提交
1043
	struct tc_htb_glob *gopt;
1044
	int err;
L
Linus Torvalds 已提交
1045
	int i;
1046 1047 1048 1049

	if (!opt)
		return -EINVAL;

1050
	err = nla_parse_nested(tb, TCA_HTB_INIT, opt, htb_policy);
1051 1052 1053
	if (err < 0)
		return err;

1054
	if (tb[TCA_HTB_INIT] == NULL) {
E
Eric Dumazet 已提交
1055
		pr_err("HTB: hey probably you have bad tc tool ?\n");
L
Linus Torvalds 已提交
1056 1057
		return -EINVAL;
	}
1058
	gopt = nla_data(tb[TCA_HTB_INIT]);
L
Linus Torvalds 已提交
1059
	if (gopt->version != HTB_VER >> 16) {
E
Eric Dumazet 已提交
1060
		pr_err("HTB: need tc/htb version %d (minor is %d), you have %d\n",
S
Stephen Hemminger 已提交
1061
		       HTB_VER >> 16, HTB_VER & 0xffff, gopt->version);
L
Linus Torvalds 已提交
1062 1063 1064
		return -EINVAL;
	}

1065 1066 1067
	err = qdisc_class_hash_init(&q->clhash);
	if (err < 0)
		return err;
L
Linus Torvalds 已提交
1068
	for (i = 0; i < TC_HTB_NUMPRIO; i++)
S
Stephen Hemminger 已提交
1069
		INIT_LIST_HEAD(q->drops + i);
L
Linus Torvalds 已提交
1070

1071
	qdisc_watchdog_init(&q->watchdog, sch);
1072
	INIT_WORK(&q->work, htb_work_func);
L
Linus Torvalds 已提交
1073 1074
	skb_queue_head_init(&q->direct_queue);

1075
	q->direct_qlen = qdisc_dev(sch)->tx_queue_len;
S
Stephen Hemminger 已提交
1076
	if (q->direct_qlen < 2)	/* some devices have zero tx_queue_len */
L
Linus Torvalds 已提交
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
		q->direct_qlen = 2;

	if ((q->rate2quantum = gopt->rate2quantum) < 1)
		q->rate2quantum = 1;
	q->defcls = gopt->defcls;

	return 0;
}

static int htb_dump(struct Qdisc *sch, struct sk_buff *skb)
{
1088
	spinlock_t *root_lock = qdisc_root_sleeping_lock(sch);
L
Linus Torvalds 已提交
1089
	struct htb_sched *q = qdisc_priv(sch);
1090
	struct nlattr *nest;
L
Linus Torvalds 已提交
1091
	struct tc_htb_glob gopt;
1092

1093
	spin_lock_bh(root_lock);
L
Linus Torvalds 已提交
1094

1095
	gopt.direct_pkts = q->direct_pkts;
L
Linus Torvalds 已提交
1096 1097 1098
	gopt.version = HTB_VER;
	gopt.rate2quantum = q->rate2quantum;
	gopt.defcls = q->defcls;
1099
	gopt.debug = 0;
1100 1101 1102 1103

	nest = nla_nest_start(skb, TCA_OPTIONS);
	if (nest == NULL)
		goto nla_put_failure;
1104 1105
	if (nla_put(skb, TCA_HTB_INIT, sizeof(gopt), &gopt))
		goto nla_put_failure;
1106 1107
	nla_nest_end(skb, nest);

1108
	spin_unlock_bh(root_lock);
L
Linus Torvalds 已提交
1109
	return skb->len;
1110

1111
nla_put_failure:
1112
	spin_unlock_bh(root_lock);
1113
	nla_nest_cancel(skb, nest);
L
Linus Torvalds 已提交
1114 1115 1116 1117
	return -1;
}

static int htb_dump_class(struct Qdisc *sch, unsigned long arg,
S
Stephen Hemminger 已提交
1118
			  struct sk_buff *skb, struct tcmsg *tcm)
L
Linus Torvalds 已提交
1119
{
S
Stephen Hemminger 已提交
1120
	struct htb_class *cl = (struct htb_class *)arg;
1121
	spinlock_t *root_lock = qdisc_root_sleeping_lock(sch);
1122
	struct nlattr *nest;
L
Linus Torvalds 已提交
1123 1124
	struct tc_htb_opt opt;

1125
	spin_lock_bh(root_lock);
1126 1127
	tcm->tcm_parent = cl->parent ? cl->parent->common.classid : TC_H_ROOT;
	tcm->tcm_handle = cl->common.classid;
L
Linus Torvalds 已提交
1128 1129 1130
	if (!cl->level && cl->un.leaf.q)
		tcm->tcm_info = cl->un.leaf.q->handle;

1131 1132 1133
	nest = nla_nest_start(skb, TCA_OPTIONS);
	if (nest == NULL)
		goto nla_put_failure;
L
Linus Torvalds 已提交
1134

S
Stephen Hemminger 已提交
1135
	memset(&opt, 0, sizeof(opt));
L
Linus Torvalds 已提交
1136

V
Vimalkumar 已提交
1137
	opt.rate.rate = cl->rate.rate_bps >> 3;
J
Jiri Pirko 已提交
1138
	opt.buffer = PSCHED_NS2TICKS(cl->buffer);
V
Vimalkumar 已提交
1139
	opt.ceil.rate = cl->ceil.rate_bps >> 3;
J
Jiri Pirko 已提交
1140
	opt.cbuffer = PSCHED_NS2TICKS(cl->cbuffer);
1141 1142
	opt.quantum = cl->quantum;
	opt.prio = cl->prio;
S
Stephen Hemminger 已提交
1143
	opt.level = cl->level;
1144 1145
	if (nla_put(skb, TCA_HTB_PARMS, sizeof(opt), &opt))
		goto nla_put_failure;
1146 1147

	nla_nest_end(skb, nest);
1148
	spin_unlock_bh(root_lock);
L
Linus Torvalds 已提交
1149
	return skb->len;
1150

1151
nla_put_failure:
1152
	spin_unlock_bh(root_lock);
1153
	nla_nest_cancel(skb, nest);
L
Linus Torvalds 已提交
1154 1155 1156 1157
	return -1;
}

static int
S
Stephen Hemminger 已提交
1158
htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
L
Linus Torvalds 已提交
1159
{
S
Stephen Hemminger 已提交
1160
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1161 1162 1163 1164 1165 1166 1167

	if (!cl->level && cl->un.leaf.q)
		cl->qstats.qlen = cl->un.leaf.q->q.qlen;
	cl->xstats.tokens = cl->tokens;
	cl->xstats.ctokens = cl->ctokens;

	if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
1168
	    gnet_stats_copy_rate_est(d, NULL, &cl->rate_est) < 0 ||
L
Linus Torvalds 已提交
1169 1170 1171 1172 1173 1174 1175
	    gnet_stats_copy_queue(d, &cl->qstats) < 0)
		return -1;

	return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));
}

static int htb_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
S
Stephen Hemminger 已提交
1176
		     struct Qdisc **old)
L
Linus Torvalds 已提交
1177
{
S
Stephen Hemminger 已提交
1178
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1179

1180 1181 1182
	if (cl->level)
		return -EINVAL;
	if (new == NULL &&
1183
	    (new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
1184 1185 1186 1187 1188 1189 1190 1191 1192
				     cl->common.classid)) == NULL)
		return -ENOBUFS;

	sch_tree_lock(sch);
	*old = cl->un.leaf.q;
	cl->un.leaf.q = new;
	if (*old != NULL) {
		qdisc_tree_decrease_qlen(*old, (*old)->q.qlen);
		qdisc_reset(*old);
L
Linus Torvalds 已提交
1193
	}
1194 1195
	sch_tree_unlock(sch);
	return 0;
L
Linus Torvalds 已提交
1196 1197
}

S
Stephen Hemminger 已提交
1198
static struct Qdisc *htb_leaf(struct Qdisc *sch, unsigned long arg)
L
Linus Torvalds 已提交
1199
{
S
Stephen Hemminger 已提交
1200
	struct htb_class *cl = (struct htb_class *)arg;
1201
	return !cl->level ? cl->un.leaf.q : NULL;
L
Linus Torvalds 已提交
1202 1203
}

1204 1205 1206 1207 1208 1209 1210 1211
static void htb_qlen_notify(struct Qdisc *sch, unsigned long arg)
{
	struct htb_class *cl = (struct htb_class *)arg;

	if (cl->un.leaf.q->q.qlen == 0)
		htb_deactivate(qdisc_priv(sch), cl);
}

L
Linus Torvalds 已提交
1212 1213
static unsigned long htb_get(struct Qdisc *sch, u32 classid)
{
S
Stephen Hemminger 已提交
1214 1215
	struct htb_class *cl = htb_find(classid, sch);
	if (cl)
L
Linus Torvalds 已提交
1216 1217 1218 1219
		cl->refcnt++;
	return (unsigned long)cl;
}

1220 1221 1222 1223 1224
static inline int htb_parent_last_child(struct htb_class *cl)
{
	if (!cl->parent)
		/* the root class */
		return 0;
1225
	if (cl->parent->children > 1)
1226 1227 1228 1229 1230
		/* not the last child */
		return 0;
	return 1;
}

1231 1232
static void htb_parent_to_leaf(struct htb_sched *q, struct htb_class *cl,
			       struct Qdisc *new_q)
1233 1234 1235
{
	struct htb_class *parent = cl->parent;

1236
	WARN_ON(cl->level || !cl->un.leaf.q || cl->prio_activity);
1237

1238 1239 1240
	if (parent->cmode != HTB_CAN_SEND)
		htb_safe_rb_erase(&parent->pq_node, q->wait_pq + parent->level);

1241 1242 1243 1244 1245 1246
	parent->level = 0;
	memset(&parent->un.inner, 0, sizeof(parent->un.inner));
	INIT_LIST_HEAD(&parent->un.leaf.drop_list);
	parent->un.leaf.q = new_q ? new_q : &noop_qdisc;
	parent->tokens = parent->buffer;
	parent->ctokens = parent->cbuffer;
1247
	parent->t_c = psched_get_time();
1248 1249 1250
	parent->cmode = HTB_CAN_SEND;
}

S
Stephen Hemminger 已提交
1251
static void htb_destroy_class(struct Qdisc *sch, struct htb_class *cl)
L
Linus Torvalds 已提交
1252 1253
{
	if (!cl->level) {
1254
		WARN_ON(!cl->un.leaf.q);
L
Linus Torvalds 已提交
1255 1256
		qdisc_destroy(cl->un.leaf.q);
	}
1257
	gen_kill_estimator(&cl->bstats, &cl->rate_est);
1258
	tcf_destroy_chain(&cl->filter_list);
L
Linus Torvalds 已提交
1259 1260 1261
	kfree(cl);
}

S
Stephen Hemminger 已提交
1262
static void htb_destroy(struct Qdisc *sch)
L
Linus Torvalds 已提交
1263 1264
{
	struct htb_sched *q = qdisc_priv(sch);
1265 1266 1267
	struct hlist_node *n, *next;
	struct htb_class *cl;
	unsigned int i;
L
Linus Torvalds 已提交
1268

1269
	cancel_work_sync(&q->work);
1270
	qdisc_watchdog_cancel(&q->watchdog);
L
Linus Torvalds 已提交
1271
	/* This line used to be after htb_destroy_class call below
E
Eric Dumazet 已提交
1272 1273 1274 1275
	 * and surprisingly it worked in 2.4. But it must precede it
	 * because filter need its target class alive to be able to call
	 * unbind_filter on it (without Oops).
	 */
1276
	tcf_destroy_chain(&q->filter_list);
S
Stephen Hemminger 已提交
1277

1278 1279
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode)
1280 1281
			tcf_destroy_chain(&cl->filter_list);
	}
1282 1283 1284
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry_safe(cl, n, next, &q->clhash.hash[i],
					  common.hnode)
1285 1286
			htb_destroy_class(sch, cl);
	}
1287
	qdisc_class_hash_destroy(&q->clhash);
L
Linus Torvalds 已提交
1288 1289 1290 1291 1292 1293
	__skb_queue_purge(&q->direct_queue);
}

static int htb_delete(struct Qdisc *sch, unsigned long arg)
{
	struct htb_sched *q = qdisc_priv(sch);
S
Stephen Hemminger 已提交
1294
	struct htb_class *cl = (struct htb_class *)arg;
1295
	unsigned int qlen;
1296 1297
	struct Qdisc *new_q = NULL;
	int last_child = 0;
L
Linus Torvalds 已提交
1298 1299 1300 1301

	// TODO: why don't allow to delete subtree ? references ? does
	// tc subsys quarantee us that in htb_destroy it holds no class
	// refs so that we can remove children safely there ?
1302
	if (cl->children || cl->filter_cnt)
L
Linus Torvalds 已提交
1303
		return -EBUSY;
S
Stephen Hemminger 已提交
1304

1305
	if (!cl->level && htb_parent_last_child(cl)) {
1306
		new_q = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
1307
					  cl->parent->common.classid);
1308 1309 1310
		last_child = 1;
	}

L
Linus Torvalds 已提交
1311
	sch_tree_lock(sch);
S
Stephen Hemminger 已提交
1312

1313
	if (!cl->level) {
1314
		qlen = cl->un.leaf.q->q.qlen;
1315
		qdisc_reset(cl->un.leaf.q);
1316
		qdisc_tree_decrease_qlen(cl->un.leaf.q, qlen);
1317 1318
	}

1319 1320
	/* delete from hash and active; remainder in destroy_class */
	qdisc_class_hash_remove(&q->clhash, &cl->common);
1321 1322
	if (cl->parent)
		cl->parent->children--;
1323

L
Linus Torvalds 已提交
1324
	if (cl->prio_activity)
S
Stephen Hemminger 已提交
1325
		htb_deactivate(q, cl);
L
Linus Torvalds 已提交
1326

1327 1328 1329
	if (cl->cmode != HTB_CAN_SEND)
		htb_safe_rb_erase(&cl->pq_node, q->wait_pq + cl->level);

1330
	if (last_child)
1331
		htb_parent_to_leaf(q, cl, new_q);
1332

1333 1334 1335 1336 1337
	BUG_ON(--cl->refcnt == 0);
	/*
	 * This shouldn't happen: we "hold" one cops->get() when called
	 * from tc_ctl_tclass; the destroy method is done from cops->put().
	 */
L
Linus Torvalds 已提交
1338 1339 1340 1341 1342 1343 1344

	sch_tree_unlock(sch);
	return 0;
}

static void htb_put(struct Qdisc *sch, unsigned long arg)
{
S
Stephen Hemminger 已提交
1345
	struct htb_class *cl = (struct htb_class *)arg;
L
Linus Torvalds 已提交
1346 1347

	if (--cl->refcnt == 0)
S
Stephen Hemminger 已提交
1348
		htb_destroy_class(sch, cl);
L
Linus Torvalds 已提交
1349 1350
}

S
Stephen Hemminger 已提交
1351
static int htb_change_class(struct Qdisc *sch, u32 classid,
1352
			    u32 parentid, struct nlattr **tca,
S
Stephen Hemminger 已提交
1353
			    unsigned long *arg)
L
Linus Torvalds 已提交
1354 1355 1356
{
	int err = -EINVAL;
	struct htb_sched *q = qdisc_priv(sch);
S
Stephen Hemminger 已提交
1357
	struct htb_class *cl = (struct htb_class *)*arg, *parent;
1358
	struct nlattr *opt = tca[TCA_OPTIONS];
1359
	struct nlattr *tb[__TCA_HTB_MAX];
L
Linus Torvalds 已提交
1360 1361 1362
	struct tc_htb_opt *hopt;

	/* extract all subattrs from opt attr */
1363 1364 1365
	if (!opt)
		goto failure;

1366
	err = nla_parse_nested(tb, TCA_HTB_MAX, opt, htb_policy);
1367 1368 1369 1370
	if (err < 0)
		goto failure;

	err = -EINVAL;
1371
	if (tb[TCA_HTB_PARMS] == NULL)
L
Linus Torvalds 已提交
1372 1373
		goto failure;

S
Stephen Hemminger 已提交
1374 1375
	parent = parentid == TC_H_ROOT ? NULL : htb_find(parentid, sch);

1376
	hopt = nla_data(tb[TCA_HTB_PARMS]);
E
Eric Dumazet 已提交
1377
	if (!hopt->rate.rate || !hopt->ceil.rate)
S
Stephen Hemminger 已提交
1378
		goto failure;
L
Linus Torvalds 已提交
1379

S
Stephen Hemminger 已提交
1380
	if (!cl) {		/* new class */
L
Linus Torvalds 已提交
1381
		struct Qdisc *new_q;
S
Stephen Hemminger 已提交
1382
		int prio;
1383
		struct {
1384
			struct nlattr		nla;
1385 1386
			struct gnet_estimator	opt;
		} est = {
1387 1388 1389
			.nla = {
				.nla_len	= nla_attr_size(sizeof(est.opt)),
				.nla_type	= TCA_RATE,
1390 1391 1392 1393 1394 1395 1396
			},
			.opt = {
				/* 4s interval, 16s averaging constant */
				.interval	= 2,
				.ewma_log	= 2,
			},
		};
S
Stephen Hemminger 已提交
1397

L
Linus Torvalds 已提交
1398
		/* check for valid classid */
1399 1400
		if (!classid || TC_H_MAJ(classid ^ sch->handle) ||
		    htb_find(classid, sch))
L
Linus Torvalds 已提交
1401 1402 1403 1404
			goto failure;

		/* check maximal depth */
		if (parent && parent->parent && parent->parent->level < 2) {
E
Eric Dumazet 已提交
1405
			pr_err("htb: tree is too deep\n");
L
Linus Torvalds 已提交
1406 1407 1408
			goto failure;
		}
		err = -ENOBUFS;
E
Eric Dumazet 已提交
1409 1410
		cl = kzalloc(sizeof(*cl), GFP_KERNEL);
		if (!cl)
L
Linus Torvalds 已提交
1411
			goto failure;
S
Stephen Hemminger 已提交
1412

1413 1414 1415 1416 1417 1418 1419 1420
		err = gen_new_estimator(&cl->bstats, &cl->rate_est,
					qdisc_root_sleeping_lock(sch),
					tca[TCA_RATE] ? : &est.nla);
		if (err) {
			kfree(cl);
			goto failure;
		}

L
Linus Torvalds 已提交
1421
		cl->refcnt = 1;
1422
		cl->children = 0;
L
Linus Torvalds 已提交
1423
		INIT_LIST_HEAD(&cl->un.leaf.drop_list);
S
Stephen Hemminger 已提交
1424 1425 1426 1427
		RB_CLEAR_NODE(&cl->pq_node);

		for (prio = 0; prio < TC_HTB_NUMPRIO; prio++)
			RB_CLEAR_NODE(&cl->node[prio]);
L
Linus Torvalds 已提交
1428 1429

		/* create leaf qdisc early because it uses kmalloc(GFP_KERNEL)
E
Eric Dumazet 已提交
1430 1431 1432
		 * so that can't be used inside of sch_tree_lock
		 * -- thanks to Karlis Peisenieks
		 */
1433
		new_q = qdisc_create_dflt(sch->dev_queue,
1434
					  &pfifo_qdisc_ops, classid);
L
Linus Torvalds 已提交
1435 1436
		sch_tree_lock(sch);
		if (parent && !parent->level) {
1437 1438
			unsigned int qlen = parent->un.leaf.q->q.qlen;

L
Linus Torvalds 已提交
1439
			/* turn parent into inner node */
1440 1441
			qdisc_reset(parent->un.leaf.q);
			qdisc_tree_decrease_qlen(parent->un.leaf.q, qlen);
S
Stephen Hemminger 已提交
1442 1443 1444
			qdisc_destroy(parent->un.leaf.q);
			if (parent->prio_activity)
				htb_deactivate(q, parent);
L
Linus Torvalds 已提交
1445 1446 1447

			/* remove from evt list because of level change */
			if (parent->cmode != HTB_CAN_SEND) {
S
Stephen Hemminger 已提交
1448
				htb_safe_rb_erase(&parent->pq_node, q->wait_pq);
L
Linus Torvalds 已提交
1449 1450 1451
				parent->cmode = HTB_CAN_SEND;
			}
			parent->level = (parent->parent ? parent->parent->level
S
Stephen Hemminger 已提交
1452 1453
					 : TC_HTB_MAXDEPTH) - 1;
			memset(&parent->un.inner, 0, sizeof(parent->un.inner));
L
Linus Torvalds 已提交
1454 1455 1456 1457
		}
		/* leaf (we) needs elementary qdisc */
		cl->un.leaf.q = new_q ? new_q : &noop_qdisc;

1458
		cl->common.classid = classid;
S
Stephen Hemminger 已提交
1459
		cl->parent = parent;
L
Linus Torvalds 已提交
1460 1461

		/* set class to be in HTB_CAN_SEND state */
1462 1463
		cl->tokens = PSCHED_TICKS2NS(hopt->buffer);
		cl->ctokens = PSCHED_TICKS2NS(hopt->cbuffer);
1464
		cl->mbuffer = 60 * PSCHED_TICKS_PER_SEC;	/* 1min */
1465
		cl->t_c = psched_get_time();
L
Linus Torvalds 已提交
1466 1467 1468
		cl->cmode = HTB_CAN_SEND;

		/* attach to the hash list and parent's family */
1469
		qdisc_class_hash_insert(&q->clhash, &cl->common);
1470 1471
		if (parent)
			parent->children++;
1472
	} else {
1473 1474 1475 1476 1477 1478 1479
		if (tca[TCA_RATE]) {
			err = gen_replace_estimator(&cl->bstats, &cl->rate_est,
						    qdisc_root_sleeping_lock(sch),
						    tca[TCA_RATE]);
			if (err)
				return err;
		}
S
Stephen Hemminger 已提交
1480
		sch_tree_lock(sch);
1481
	}
L
Linus Torvalds 已提交
1482 1483

	/* it used to be a nasty bug here, we have to check that node
E
Eric Dumazet 已提交
1484 1485
	 * is really leaf before changing cl->un.leaf !
	 */
L
Linus Torvalds 已提交
1486
	if (!cl->level) {
E
Eric Dumazet 已提交
1487
		cl->quantum = hopt->rate.rate / q->rate2quantum;
1488
		if (!hopt->quantum && cl->quantum < 1000) {
E
Eric Dumazet 已提交
1489
			pr_warning(
S
Stephen Hemminger 已提交
1490
			       "HTB: quantum of class %X is small. Consider r2q change.\n",
1491
			       cl->common.classid);
1492
			cl->quantum = 1000;
L
Linus Torvalds 已提交
1493
		}
1494
		if (!hopt->quantum && cl->quantum > 200000) {
E
Eric Dumazet 已提交
1495
			pr_warning(
S
Stephen Hemminger 已提交
1496
			       "HTB: quantum of class %X is big. Consider r2q change.\n",
1497
			       cl->common.classid);
1498
			cl->quantum = 200000;
L
Linus Torvalds 已提交
1499 1500
		}
		if (hopt->quantum)
1501 1502 1503
			cl->quantum = hopt->quantum;
		if ((cl->prio = hopt->prio) >= TC_HTB_NUMPRIO)
			cl->prio = TC_HTB_NUMPRIO - 1;
L
Linus Torvalds 已提交
1504 1505
	}

E
Eric Dumazet 已提交
1506 1507
	cl->rate.rate_bps = (u64)hopt->rate.rate << 3;
	cl->ceil.rate_bps = (u64)hopt->ceil.rate << 3;
V
Vimalkumar 已提交
1508 1509 1510 1511

	htb_precompute_ratedata(&cl->rate);
	htb_precompute_ratedata(&cl->ceil);

J
Jiri Pirko 已提交
1512 1513
	cl->buffer = PSCHED_TICKS2NS(hopt->buffer);
	cl->cbuffer = PSCHED_TICKS2NS(hopt->buffer);
V
Vimalkumar 已提交
1514

L
Linus Torvalds 已提交
1515 1516
	sch_tree_unlock(sch);

1517 1518
	qdisc_class_hash_grow(sch, &q->clhash);

L
Linus Torvalds 已提交
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530
	*arg = (unsigned long)cl;
	return 0;

failure:
	return err;
}

static struct tcf_proto **htb_find_tcf(struct Qdisc *sch, unsigned long arg)
{
	struct htb_sched *q = qdisc_priv(sch);
	struct htb_class *cl = (struct htb_class *)arg;
	struct tcf_proto **fl = cl ? &cl->filter_list : &q->filter_list;
1531

L
Linus Torvalds 已提交
1532 1533 1534 1535
	return fl;
}

static unsigned long htb_bind_filter(struct Qdisc *sch, unsigned long parent,
S
Stephen Hemminger 已提交
1536
				     u32 classid)
L
Linus Torvalds 已提交
1537
{
S
Stephen Hemminger 已提交
1538
	struct htb_class *cl = htb_find(classid, sch);
1539

L
Linus Torvalds 已提交
1540
	/*if (cl && !cl->level) return 0;
E
Eric Dumazet 已提交
1541 1542 1543 1544 1545 1546 1547
	 * The line above used to be there to prevent attaching filters to
	 * leaves. But at least tc_index filter uses this just to get class
	 * for other reasons so that we have to allow for it.
	 * ----
	 * 19.6.2002 As Werner explained it is ok - bind filter is just
	 * another way to "lock" the class - unlike "get" this lock can
	 * be broken by class during destroy IIUC.
L
Linus Torvalds 已提交
1548
	 */
S
Stephen Hemminger 已提交
1549 1550
	if (cl)
		cl->filter_cnt++;
L
Linus Torvalds 已提交
1551 1552 1553 1554 1555 1556
	return (unsigned long)cl;
}

static void htb_unbind_filter(struct Qdisc *sch, unsigned long arg)
{
	struct htb_class *cl = (struct htb_class *)arg;
1557

S
Stephen Hemminger 已提交
1558 1559
	if (cl)
		cl->filter_cnt--;
L
Linus Torvalds 已提交
1560 1561 1562 1563 1564
}

static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
	struct htb_sched *q = qdisc_priv(sch);
1565 1566 1567
	struct htb_class *cl;
	struct hlist_node *n;
	unsigned int i;
L
Linus Torvalds 已提交
1568 1569 1570 1571

	if (arg->stop)
		return;

1572 1573
	for (i = 0; i < q->clhash.hashsize; i++) {
		hlist_for_each_entry(cl, n, &q->clhash.hash[i], common.hnode) {
L
Linus Torvalds 已提交
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586
			if (arg->count < arg->skip) {
				arg->count++;
				continue;
			}
			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
				arg->stop = 1;
				return;
			}
			arg->count++;
		}
	}
}

1587
static const struct Qdisc_class_ops htb_class_ops = {
L
Linus Torvalds 已提交
1588 1589
	.graft		=	htb_graft,
	.leaf		=	htb_leaf,
1590
	.qlen_notify	=	htb_qlen_notify,
L
Linus Torvalds 已提交
1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
	.get		=	htb_get,
	.put		=	htb_put,
	.change		=	htb_change_class,
	.delete		=	htb_delete,
	.walk		=	htb_walk,
	.tcf_chain	=	htb_find_tcf,
	.bind_tcf	=	htb_bind_filter,
	.unbind_tcf	=	htb_unbind_filter,
	.dump		=	htb_dump_class,
	.dump_stats	=	htb_dump_class_stats,
};

1603
static struct Qdisc_ops htb_qdisc_ops __read_mostly = {
L
Linus Torvalds 已提交
1604 1605 1606 1607 1608
	.cl_ops		=	&htb_class_ops,
	.id		=	"htb",
	.priv_size	=	sizeof(struct htb_sched),
	.enqueue	=	htb_enqueue,
	.dequeue	=	htb_dequeue,
1609
	.peek		=	qdisc_peek_dequeued,
L
Linus Torvalds 已提交
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619
	.drop		=	htb_drop,
	.init		=	htb_init,
	.reset		=	htb_reset,
	.destroy	=	htb_destroy,
	.dump		=	htb_dump,
	.owner		=	THIS_MODULE,
};

static int __init htb_module_init(void)
{
S
Stephen Hemminger 已提交
1620
	return register_qdisc(&htb_qdisc_ops);
L
Linus Torvalds 已提交
1621
}
S
Stephen Hemminger 已提交
1622
static void __exit htb_module_exit(void)
L
Linus Torvalds 已提交
1623
{
S
Stephen Hemminger 已提交
1624
	unregister_qdisc(&htb_qdisc_ops);
L
Linus Torvalds 已提交
1625
}
S
Stephen Hemminger 已提交
1626

L
Linus Torvalds 已提交
1627 1628 1629
module_init(htb_module_init)
module_exit(htb_module_exit)
MODULE_LICENSE("GPL");