sch_sfq.c 14.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
 * net/sched/sch_sfq.c	Stochastic Fairness Queueing discipline.
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
 *
 * Authors:	Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/jiffies.h>
#include <linux/string.h>
#include <linux/in.h>
#include <linux/errno.h>
#include <linux/init.h>
#include <linux/ipv6.h>
#include <linux/skbuff.h>
22
#include <linux/jhash.h>
23
#include <linux/slab.h>
24 25
#include <net/ip.h>
#include <net/netlink.h>
L
Linus Torvalds 已提交
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
#include <net/pkt_sched.h>


/*	Stochastic Fairness Queuing algorithm.
	=======================================

	Source:
	Paul E. McKenney "Stochastic Fairness Queuing",
	IEEE INFOCOMM'90 Proceedings, San Francisco, 1990.

	Paul E. McKenney "Stochastic Fairness Queuing",
	"Interworking: Research and Experience", v.2, 1991, p.113-131.


	See also:
	M. Shreedhar and George Varghese "Efficient Fair
	Queuing using Deficit Round Robin", Proc. SIGCOMM 95.


45
	This is not the thing that is usually called (W)FQ nowadays.
L
Linus Torvalds 已提交
46 47 48 49 50 51 52 53 54
	It does not use any timestamp mechanism, but instead
	processes queues in round-robin order.

	ADVANTAGE:

	- It is very cheap. Both CPU and memory requirements are minimal.

	DRAWBACKS:

55
	- "Stochastic" -> It is not 100% fair.
L
Linus Torvalds 已提交
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
	When hash collisions occur, several flows are considered as one.

	- "Round-robin" -> It introduces larger delays than virtual clock
	based schemes, and should not be used for isolating interactive
	traffic	from non-interactive. It means, that this scheduler
	should be used as leaf of CBQ or P3, which put interactive traffic
	to higher priority band.

	We still need true WFQ for top level CSZ, but using WFQ
	for the best effort traffic is absolutely pointless:
	SFQ is superior for this purpose.

	IMPLEMENTATION:
	This implementation limits maximal queue length to 128;
	maximal mtu to 2^15-1; number of hash buckets to 1024.
	The only goal of this restrictions was that all data
	fit into one 4K page :-). Struct sfq_sched_data is
	organized in anti-cache manner: all the data for a bucket
	are scattered over different locations. This is not good,
	but it allowed me to put it into 4K.

	It is easy to increase these values, but not in flight.  */

#define SFQ_DEPTH		128
#define SFQ_HASH_DIVISOR	1024

/* This type should contain at least SFQ_DEPTH*2 values */
typedef unsigned char sfq_index;

struct sfq_head
{
	sfq_index	next;
	sfq_index	prev;
};

struct sfq_sched_data
{
/* Parameters */
	int		perturb_period;
	unsigned	quantum;	/* Allotment per round: MUST BE >= MTU */
	int		limit;

/* Variables */
99
	struct tcf_proto *filter_list;
L
Linus Torvalds 已提交
100
	struct timer_list perturb_timer;
101
	u32		perturbation;
L
Linus Torvalds 已提交
102 103 104 105 106 107 108 109 110 111 112 113 114
	sfq_index	tail;		/* Index of current slot in round */
	sfq_index	max_depth;	/* Maximal depth */

	sfq_index	ht[SFQ_HASH_DIVISOR];	/* Hash table */
	sfq_index	next[SFQ_DEPTH];	/* Active slots link */
	short		allot[SFQ_DEPTH];	/* Current allotment per slot */
	unsigned short	hash[SFQ_DEPTH];	/* Hash value indexed by slots */
	struct sk_buff_head	qs[SFQ_DEPTH];		/* Slot queue */
	struct sfq_head	dep[SFQ_DEPTH*2];	/* Linked list of slots, indexed by depth */
};

static __inline__ unsigned sfq_fold_hash(struct sfq_sched_data *q, u32 h, u32 h1)
{
115
	return jhash_2words(h, h1, q->perturbation) & (SFQ_HASH_DIVISOR - 1);
L
Linus Torvalds 已提交
116 117 118 119 120 121 122
}

static unsigned sfq_hash(struct sfq_sched_data *q, struct sk_buff *skb)
{
	u32 h, h2;

	switch (skb->protocol) {
123
	case htons(ETH_P_IP):
L
Linus Torvalds 已提交
124
	{
125
		const struct iphdr *iph;
126
		int poff;
127 128 129 130

		if (!pskb_network_may_pull(skb, sizeof(*iph)))
			goto err;
		iph = ip_hdr(skb);
131 132
		h = (__force u32)iph->daddr;
		h2 = (__force u32)iph->saddr ^ iph->protocol;
133 134 135 136 137 138 139 140
		if (iph->frag_off & htons(IP_MF|IP_OFFSET))
			break;
		poff = proto_ports_offset(iph->protocol);
		if (poff >= 0 &&
		    pskb_network_may_pull(skb, iph->ihl * 4 + 4 + poff)) {
			iph = ip_hdr(skb);
			h2 ^= *(u32*)((void *)iph + iph->ihl * 4 + poff);
		}
L
Linus Torvalds 已提交
141 142
		break;
	}
143
	case htons(ETH_P_IPV6):
L
Linus Torvalds 已提交
144
	{
145
		struct ipv6hdr *iph;
146
		int poff;
147 148 149 150

		if (!pskb_network_may_pull(skb, sizeof(*iph)))
			goto err;
		iph = ipv6_hdr(skb);
151 152
		h = (__force u32)iph->daddr.s6_addr32[3];
		h2 = (__force u32)iph->saddr.s6_addr32[3] ^ iph->nexthdr;
153 154 155 156 157 158
		poff = proto_ports_offset(iph->nexthdr);
		if (poff >= 0 &&
		    pskb_network_may_pull(skb, sizeof(*iph) + 4 + poff)) {
			iph = ipv6_hdr(skb);
			h2 ^= *(u32*)((void *)iph + sizeof(*iph) + poff);
		}
L
Linus Torvalds 已提交
159 160 161
		break;
	}
	default:
162
err:
163
		h = (unsigned long)skb_dst(skb) ^ (__force u32)skb->protocol;
164
		h2 = (unsigned long)skb->sk;
L
Linus Torvalds 已提交
165
	}
166

L
Linus Torvalds 已提交
167 168 169
	return sfq_fold_hash(q, h, h2);
}

170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
static unsigned int sfq_classify(struct sk_buff *skb, struct Qdisc *sch,
				 int *qerr)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	struct tcf_result res;
	int result;

	if (TC_H_MAJ(skb->priority) == sch->handle &&
	    TC_H_MIN(skb->priority) > 0 &&
	    TC_H_MIN(skb->priority) <= SFQ_HASH_DIVISOR)
		return TC_H_MIN(skb->priority);

	if (!q->filter_list)
		return sfq_hash(q, skb) + 1;

185
	*qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
186 187 188 189 190 191
	result = tc_classify(skb, q->filter_list, &res);
	if (result >= 0) {
#ifdef CONFIG_NET_CLS_ACT
		switch (result) {
		case TC_ACT_STOLEN:
		case TC_ACT_QUEUED:
192
			*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
193 194 195 196 197 198 199 200 201 202
		case TC_ACT_SHOT:
			return 0;
		}
#endif
		if (TC_H_MIN(res.classid) <= SFQ_HASH_DIVISOR)
			return TC_H_MIN(res.classid);
	}
	return 0;
}

L
Linus Torvalds 已提交
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
static inline void sfq_link(struct sfq_sched_data *q, sfq_index x)
{
	sfq_index p, n;
	int d = q->qs[x].qlen + SFQ_DEPTH;

	p = d;
	n = q->dep[d].next;
	q->dep[x].next = n;
	q->dep[x].prev = p;
	q->dep[p].next = q->dep[n].prev = x;
}

static inline void sfq_dec(struct sfq_sched_data *q, sfq_index x)
{
	sfq_index p, n;

	n = q->dep[x].next;
	p = q->dep[x].prev;
	q->dep[p].next = n;
	q->dep[n].prev = p;

	if (n == p && q->max_depth == q->qs[x].qlen + 1)
		q->max_depth--;

	sfq_link(q, x);
}

static inline void sfq_inc(struct sfq_sched_data *q, sfq_index x)
{
	sfq_index p, n;
	int d;

	n = q->dep[x].next;
	p = q->dep[x].prev;
	q->dep[p].next = n;
	q->dep[n].prev = p;
	d = q->qs[x].qlen;
	if (q->max_depth < d)
		q->max_depth = d;

	sfq_link(q, x);
}

static unsigned int sfq_drop(struct Qdisc *sch)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	sfq_index d = q->max_depth;
	struct sk_buff *skb;
	unsigned int len;

	/* Queue is full! Find the longest slot and
	   drop a packet from it */

	if (d > 1) {
257
		sfq_index x = q->dep[d + SFQ_DEPTH].next;
L
Linus Torvalds 已提交
258
		skb = q->qs[x].prev;
259
		len = qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
260 261 262 263 264
		__skb_unlink(skb, &q->qs[x]);
		kfree_skb(skb);
		sfq_dec(q, x);
		sch->q.qlen--;
		sch->qstats.drops++;
265
		sch->qstats.backlog -= len;
L
Linus Torvalds 已提交
266 267 268 269 270 271 272 273
		return len;
	}

	if (d == 1) {
		/* It is difficult to believe, but ALL THE SLOTS HAVE LENGTH 1. */
		d = q->next[q->tail];
		q->next[q->tail] = q->next[d];
		skb = q->qs[d].prev;
274
		len = qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
275 276 277 278 279 280
		__skb_unlink(skb, &q->qs[d]);
		kfree_skb(skb);
		sfq_dec(q, d);
		sch->q.qlen--;
		q->ht[q->hash[d]] = SFQ_DEPTH;
		sch->qstats.drops++;
281
		sch->qstats.backlog -= len;
L
Linus Torvalds 已提交
282 283 284 285 286 287 288
		return len;
	}

	return 0;
}

static int
289
sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
L
Linus Torvalds 已提交
290 291
{
	struct sfq_sched_data *q = qdisc_priv(sch);
292
	unsigned int hash;
L
Linus Torvalds 已提交
293
	sfq_index x;
294
	int uninitialized_var(ret);
295 296 297

	hash = sfq_classify(skb, sch, &ret);
	if (hash == 0) {
298
		if (ret & __NET_XMIT_BYPASS)
299 300 301 302 303
			sch->qstats.drops++;
		kfree_skb(skb);
		return ret;
	}
	hash--;
L
Linus Torvalds 已提交
304 305 306 307 308 309

	x = q->ht[hash];
	if (x == SFQ_DEPTH) {
		q->ht[hash] = x = q->dep[SFQ_DEPTH].next;
		q->hash[x] = hash;
	}
310

311 312 313 314 315 316 317
	/* If selected queue has length q->limit, this means that
	 * all another queues are empty and that we do simple tail drop,
	 * i.e. drop _this_ packet.
	 */
	if (q->qs[x].qlen >= q->limit)
		return qdisc_drop(skb, sch);

318
	sch->qstats.backlog += qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
319 320 321 322 323 324 325 326 327
	__skb_queue_tail(&q->qs[x], skb);
	sfq_inc(q, x);
	if (q->qs[x].qlen == 1) {		/* The flow is new */
		if (q->tail == SFQ_DEPTH) {	/* It is the first flow */
			q->next[x] = x;
		} else {
			q->next[x] = q->next[q->tail];
			q->next[q->tail] = x;
		}
328 329
		q->tail = x;
		q->allot[x] = q->quantum;
L
Linus Torvalds 已提交
330
	}
331
	if (++sch->q.qlen <= q->limit) {
332
		sch->bstats.bytes += qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
333
		sch->bstats.packets++;
334
		return NET_XMIT_SUCCESS;
L
Linus Torvalds 已提交
335 336 337 338 339 340
	}

	sfq_drop(sch);
	return NET_XMIT_CN;
}

341 342 343 344 345
static struct sk_buff *
sfq_peek(struct Qdisc *sch)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	sfq_index a;
L
Linus Torvalds 已提交
346

347 348 349
	/* No active slots */
	if (q->tail == SFQ_DEPTH)
		return NULL;
L
Linus Torvalds 已提交
350

351 352 353
	a = q->next[q->tail];
	return skb_peek(&q->qs[a]);
}
L
Linus Torvalds 已提交
354 355

static struct sk_buff *
356
sfq_dequeue(struct Qdisc *sch)
L
Linus Torvalds 已提交
357 358 359
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	struct sk_buff *skb;
360
	sfq_index a, next_a;
L
Linus Torvalds 已提交
361 362 363 364 365

	/* No active slots */
	if (q->tail == SFQ_DEPTH)
		return NULL;

366
	a = q->next[q->tail];
L
Linus Torvalds 已提交
367 368 369 370 371

	/* Grab packet */
	skb = __skb_dequeue(&q->qs[a]);
	sfq_dec(q, a);
	sch->q.qlen--;
372
	sch->qstats.backlog -= qdisc_pkt_len(skb);
L
Linus Torvalds 已提交
373 374 375 376

	/* Is the slot empty? */
	if (q->qs[a].qlen == 0) {
		q->ht[q->hash[a]] = SFQ_DEPTH;
377 378
		next_a = q->next[a];
		if (a == next_a) {
L
Linus Torvalds 已提交
379 380 381
			q->tail = SFQ_DEPTH;
			return skb;
		}
382
		q->next[q->tail] = next_a;
383
	} else if ((q->allot[a] -= qdisc_pkt_len(skb)) <= 0) {
L
Linus Torvalds 已提交
384
		q->allot[a] += q->quantum;
385
		q->tail = a;
L
Linus Torvalds 已提交
386 387 388 389 390
	}
	return skb;
}

static void
391
sfq_reset(struct Qdisc *sch)
L
Linus Torvalds 已提交
392 393 394 395 396 397 398 399 400
{
	struct sk_buff *skb;

	while ((skb = sfq_dequeue(sch)) != NULL)
		kfree_skb(skb);
}

static void sfq_perturbation(unsigned long arg)
{
401
	struct Qdisc *sch = (struct Qdisc *)arg;
L
Linus Torvalds 已提交
402 403
	struct sfq_sched_data *q = qdisc_priv(sch);

404
	q->perturbation = net_random();
L
Linus Torvalds 已提交
405

406 407
	if (q->perturb_period)
		mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
L
Linus Torvalds 已提交
408 409
}

410
static int sfq_change(struct Qdisc *sch, struct nlattr *opt)
L
Linus Torvalds 已提交
411 412
{
	struct sfq_sched_data *q = qdisc_priv(sch);
413
	struct tc_sfq_qopt *ctl = nla_data(opt);
414
	unsigned int qlen;
L
Linus Torvalds 已提交
415

416
	if (opt->nla_len < nla_attr_size(sizeof(*ctl)))
L
Linus Torvalds 已提交
417 418 419
		return -EINVAL;

	sch_tree_lock(sch);
420
	q->quantum = ctl->quantum ? : psched_mtu(qdisc_dev(sch));
421
	q->perturb_period = ctl->perturb_period * HZ;
L
Linus Torvalds 已提交
422
	if (ctl->limit)
423
		q->limit = min_t(u32, ctl->limit, SFQ_DEPTH - 1);
L
Linus Torvalds 已提交
424

425
	qlen = sch->q.qlen;
426
	while (sch->q.qlen > q->limit)
L
Linus Torvalds 已提交
427
		sfq_drop(sch);
428
	qdisc_tree_decrease_qlen(sch, qlen - sch->q.qlen);
L
Linus Torvalds 已提交
429 430 431

	del_timer(&q->perturb_timer);
	if (q->perturb_period) {
432
		mod_timer(&q->perturb_timer, jiffies + q->perturb_period);
433
		q->perturbation = net_random();
L
Linus Torvalds 已提交
434 435 436 437 438
	}
	sch_tree_unlock(sch);
	return 0;
}

439
static int sfq_init(struct Qdisc *sch, struct nlattr *opt)
L
Linus Torvalds 已提交
440 441 442 443
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	int i;

444
	q->perturb_timer.function = sfq_perturbation;
445
	q->perturb_timer.data = (unsigned long)sch;
446
	init_timer_deferrable(&q->perturb_timer);
L
Linus Torvalds 已提交
447

448
	for (i = 0; i < SFQ_HASH_DIVISOR; i++)
L
Linus Torvalds 已提交
449
		q->ht[i] = SFQ_DEPTH;
450 451

	for (i = 0; i < SFQ_DEPTH; i++) {
L
Linus Torvalds 已提交
452
		skb_queue_head_init(&q->qs[i]);
453 454
		q->dep[i + SFQ_DEPTH].next = i + SFQ_DEPTH;
		q->dep[i + SFQ_DEPTH].prev = i + SFQ_DEPTH;
L
Linus Torvalds 已提交
455
	}
456

457
	q->limit = SFQ_DEPTH - 1;
L
Linus Torvalds 已提交
458 459 460
	q->max_depth = 0;
	q->tail = SFQ_DEPTH;
	if (opt == NULL) {
461
		q->quantum = psched_mtu(qdisc_dev(sch));
L
Linus Torvalds 已提交
462
		q->perturb_period = 0;
463
		q->perturbation = net_random();
L
Linus Torvalds 已提交
464 465 466 467 468
	} else {
		int err = sfq_change(sch, opt);
		if (err)
			return err;
	}
469 470

	for (i = 0; i < SFQ_DEPTH; i++)
L
Linus Torvalds 已提交
471 472 473 474 475 476 477
		sfq_link(q, i);
	return 0;
}

static void sfq_destroy(struct Qdisc *sch)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
478

479
	tcf_destroy_chain(&q->filter_list);
480 481
	q->perturb_period = 0;
	del_timer_sync(&q->perturb_timer);
L
Linus Torvalds 已提交
482 483 484 485 486
}

static int sfq_dump(struct Qdisc *sch, struct sk_buff *skb)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
487
	unsigned char *b = skb_tail_pointer(skb);
L
Linus Torvalds 已提交
488 489 490
	struct tc_sfq_qopt opt;

	opt.quantum = q->quantum;
491
	opt.perturb_period = q->perturb_period / HZ;
L
Linus Torvalds 已提交
492 493 494

	opt.limit = q->limit;
	opt.divisor = SFQ_HASH_DIVISOR;
495
	opt.flows = q->limit;
L
Linus Torvalds 已提交
496

497
	NLA_PUT(skb, TCA_OPTIONS, sizeof(opt), &opt);
L
Linus Torvalds 已提交
498 499 500

	return skb->len;

501
nla_put_failure:
502
	nlmsg_trim(skb, b);
L
Linus Torvalds 已提交
503 504 505
	return -1;
}

506 507 508 509 510
static struct Qdisc *sfq_leaf(struct Qdisc *sch, unsigned long arg)
{
	return NULL;
}

511 512 513 514 515
static unsigned long sfq_get(struct Qdisc *sch, u32 classid)
{
	return 0;
}

516 517 518 519 520 521
static unsigned long sfq_bind(struct Qdisc *sch, unsigned long parent,
			      u32 classid)
{
	return 0;
}

522 523 524 525
static void sfq_put(struct Qdisc *q, unsigned long cl)
{
}

526 527 528 529 530 531 532 533 534
static struct tcf_proto **sfq_find_tcf(struct Qdisc *sch, unsigned long cl)
{
	struct sfq_sched_data *q = qdisc_priv(sch);

	if (cl)
		return NULL;
	return &q->filter_list;
}

535 536 537 538 539 540 541 542 543 544 545 546
static int sfq_dump_class(struct Qdisc *sch, unsigned long cl,
			  struct sk_buff *skb, struct tcmsg *tcm)
{
	tcm->tcm_handle |= TC_H_MIN(cl);
	return 0;
}

static int sfq_dump_class_stats(struct Qdisc *sch, unsigned long cl,
				struct gnet_dump *d)
{
	struct sfq_sched_data *q = qdisc_priv(sch);
	sfq_index idx = q->ht[cl-1];
547 548
	struct sk_buff_head *list = &q->qs[idx];
	struct gnet_stats_queue qs = { .qlen = list->qlen };
549
	struct tc_sfq_xstats xstats = { .allot = q->allot[idx] };
550 551 552 553
	struct sk_buff *skb;

	skb_queue_walk(list, skb)
		qs.backlog += qdisc_pkt_len(skb);
554 555 556 557 558 559

	if (gnet_stats_copy_queue(d, &qs) < 0)
		return -1;
	return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
}

560 561
static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
{
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
	struct sfq_sched_data *q = qdisc_priv(sch);
	unsigned int i;

	if (arg->stop)
		return;

	for (i = 0; i < SFQ_HASH_DIVISOR; i++) {
		if (q->ht[i] == SFQ_DEPTH ||
		    arg->count < arg->skip) {
			arg->count++;
			continue;
		}
		if (arg->fn(sch, i + 1, arg) < 0) {
			arg->stop = 1;
			break;
		}
		arg->count++;
	}
580 581 582
}

static const struct Qdisc_class_ops sfq_class_ops = {
583
	.leaf		=	sfq_leaf,
584
	.get		=	sfq_get,
585
	.put		=	sfq_put,
586
	.tcf_chain	=	sfq_find_tcf,
587
	.bind_tcf	=	sfq_bind,
588
	.unbind_tcf	=	sfq_put,
589 590
	.dump		=	sfq_dump_class,
	.dump_stats	=	sfq_dump_class_stats,
591 592 593
	.walk		=	sfq_walk,
};

594
static struct Qdisc_ops sfq_qdisc_ops __read_mostly = {
595
	.cl_ops		=	&sfq_class_ops,
L
Linus Torvalds 已提交
596 597 598 599
	.id		=	"sfq",
	.priv_size	=	sizeof(struct sfq_sched_data),
	.enqueue	=	sfq_enqueue,
	.dequeue	=	sfq_dequeue,
600
	.peek		=	sfq_peek,
L
Linus Torvalds 已提交
601 602 603 604 605 606 607 608 609 610 611 612 613
	.drop		=	sfq_drop,
	.init		=	sfq_init,
	.reset		=	sfq_reset,
	.destroy	=	sfq_destroy,
	.change		=	NULL,
	.dump		=	sfq_dump,
	.owner		=	THIS_MODULE,
};

static int __init sfq_module_init(void)
{
	return register_qdisc(&sfq_qdisc_ops);
}
614
static void __exit sfq_module_exit(void)
L
Linus Torvalds 已提交
615 616 617 618 619 620
{
	unregister_qdisc(&sfq_qdisc_ops);
}
module_init(sfq_module_init)
module_exit(sfq_module_exit)
MODULE_LICENSE("GPL");