af_netlink.c 67.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * NETLINK      Kernel-user communication protocol.
 *
4
 * 		Authors:	Alan Cox <alan@lxorguk.ukuu.org.uk>
L
Linus Torvalds 已提交
5
 * 				Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
6
 * 				Patrick McHardy <kaber@trash.net>
L
Linus Torvalds 已提交
7 8 9 10 11
 *
 *		This program is free software; you can redistribute it and/or
 *		modify it under the terms of the GNU General Public License
 *		as published by the Free Software Foundation; either version
 *		2 of the License, or (at your option) any later version.
12
 *
L
Linus Torvalds 已提交
13 14 15 16
 * Tue Jun 26 14:36:48 MEST 2001 Herbert "herp" Rosmanith
 *                               added netlink_proto_exit
 * Tue Jan 22 18:32:44 BRST 2002 Arnaldo C. de Melo <acme@conectiva.com.br>
 * 				 use nlk_sk, as sk->protinfo is on a diet 8)
17 18 19 20 21 22
 * Fri Jul 22 19:51:12 MEST 2005 Harald Welte <laforge@gnumonks.org>
 * 				 - inc module use count of module that owns
 * 				   the kernel socket in case userspace opens
 * 				   socket of same protocol
 * 				 - remove all module support, since netlink is
 * 				   mandatory if CONFIG_NET=y these days
L
Linus Torvalds 已提交
23 24 25 26
 */

#include <linux/module.h>

27
#include <linux/capability.h>
L
Linus Torvalds 已提交
28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/stat.h>
#include <linux/socket.h>
#include <linux/un.h>
#include <linux/fcntl.h>
#include <linux/termios.h>
#include <linux/sockios.h>
#include <linux/net.h>
#include <linux/fs.h>
#include <linux/slab.h>
#include <asm/uaccess.h>
#include <linux/skbuff.h>
#include <linux/netdevice.h>
#include <linux/rtnetlink.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/notifier.h>
#include <linux/security.h>
#include <linux/jhash.h>
#include <linux/jiffies.h>
#include <linux/random.h>
#include <linux/bitops.h>
#include <linux/mm.h>
#include <linux/types.h>
A
Andrew Morton 已提交
57
#include <linux/audit.h>
58
#include <linux/mutex.h>
59
#include <linux/vmalloc.h>
60
#include <asm/cacheflush.h>
A
Andrew Morton 已提交
61

62
#include <net/net_namespace.h>
L
Linus Torvalds 已提交
63 64
#include <net/sock.h>
#include <net/scm.h>
65
#include <net/netlink.h>
L
Linus Torvalds 已提交
66

67
#include "af_netlink.h"
L
Linus Torvalds 已提交
68

69 70 71
struct listeners {
	struct rcu_head		rcu;
	unsigned long		masks[0];
72 73
};

74 75 76 77
/* state bits */
#define NETLINK_CONGESTED	0x0

/* flags */
78
#define NETLINK_KERNEL_SOCKET	0x1
79
#define NETLINK_RECV_PKTINFO	0x2
80
#define NETLINK_BROADCAST_SEND_ERROR	0x4
81
#define NETLINK_RECV_NO_ENOBUFS	0x8
82

83
static inline int netlink_is_kernel(struct sock *sk)
84 85 86 87
{
	return nlk_sk(sk)->flags & NETLINK_KERNEL_SOCKET;
}

88 89
struct netlink_table *nl_table;
EXPORT_SYMBOL_GPL(nl_table);
L
Linus Torvalds 已提交
90 91 92 93

static DECLARE_WAIT_QUEUE_HEAD(nl_table_wait);

static int netlink_dump(struct sock *sk);
94
static void netlink_skb_destructor(struct sk_buff *skb);
L
Linus Torvalds 已提交
95

96 97
DEFINE_RWLOCK(nl_table_lock);
EXPORT_SYMBOL_GPL(nl_table_lock);
L
Linus Torvalds 已提交
98 99
static atomic_t nl_table_users = ATOMIC_INIT(0);

100 101
#define nl_deref_protected(X) rcu_dereference_protected(X, lockdep_is_held(&nl_table_lock));

102
static ATOMIC_NOTIFIER_HEAD(netlink_chain);
L
Linus Torvalds 已提交
103

104
static inline u32 netlink_group_mask(u32 group)
105 106 107 108
{
	return group ? 1 << (group - 1) : 0;
}

109
static inline struct hlist_head *nl_portid_hashfn(struct nl_portid_hash *hash, u32 portid)
L
Linus Torvalds 已提交
110
{
111
	return &hash->table[jhash_1word(portid, hash->rnd) & hash->mask];
L
Linus Torvalds 已提交
112 113
}

114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
static void netlink_overrun(struct sock *sk)
{
	struct netlink_sock *nlk = nlk_sk(sk);

	if (!(nlk->flags & NETLINK_RECV_NO_ENOBUFS)) {
		if (!test_and_set_bit(NETLINK_CONGESTED, &nlk_sk(sk)->state)) {
			sk->sk_err = ENOBUFS;
			sk->sk_error_report(sk);
		}
	}
	atomic_inc(&sk->sk_drops);
}

static void netlink_rcv_wake(struct sock *sk)
{
	struct netlink_sock *nlk = nlk_sk(sk);

	if (skb_queue_empty(&sk->sk_receive_queue))
		clear_bit(NETLINK_CONGESTED, &nlk->state);
	if (!test_bit(NETLINK_CONGESTED, &nlk->state))
		wake_up_interruptible(&nlk->wait);
}

137
#ifdef CONFIG_NETLINK_MMAP
138 139 140 141 142
static bool netlink_skb_is_mmaped(const struct sk_buff *skb)
{
	return NETLINK_CB(skb).flags & NETLINK_SKB_MMAPED;
}

143 144 145 146 147
static bool netlink_rx_is_mmaped(struct sock *sk)
{
	return nlk_sk(sk)->rx_ring.pg_vec != NULL;
}

148 149 150 151 152
static bool netlink_tx_is_mmaped(struct sock *sk)
{
	return nlk_sk(sk)->tx_ring.pg_vec != NULL;
}

153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
static __pure struct page *pgvec_to_page(const void *addr)
{
	if (is_vmalloc_addr(addr))
		return vmalloc_to_page(addr);
	else
		return virt_to_page(addr);
}

static void free_pg_vec(void **pg_vec, unsigned int order, unsigned int len)
{
	unsigned int i;

	for (i = 0; i < len; i++) {
		if (pg_vec[i] != NULL) {
			if (is_vmalloc_addr(pg_vec[i]))
				vfree(pg_vec[i]);
			else
				free_pages((unsigned long)pg_vec[i], order);
		}
	}
	kfree(pg_vec);
}

static void *alloc_one_pg_vec_page(unsigned long order)
{
	void *buffer;
	gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO |
			  __GFP_NOWARN | __GFP_NORETRY;

	buffer = (void *)__get_free_pages(gfp_flags, order);
	if (buffer != NULL)
		return buffer;

	buffer = vzalloc((1 << order) * PAGE_SIZE);
	if (buffer != NULL)
		return buffer;

	gfp_flags &= ~__GFP_NORETRY;
	return (void *)__get_free_pages(gfp_flags, order);
}

static void **alloc_pg_vec(struct netlink_sock *nlk,
			   struct nl_mmap_req *req, unsigned int order)
{
	unsigned int block_nr = req->nm_block_nr;
	unsigned int i;
	void **pg_vec, *ptr;

	pg_vec = kcalloc(block_nr, sizeof(void *), GFP_KERNEL);
	if (pg_vec == NULL)
		return NULL;

	for (i = 0; i < block_nr; i++) {
		pg_vec[i] = ptr = alloc_one_pg_vec_page(order);
		if (pg_vec[i] == NULL)
			goto err1;
	}

	return pg_vec;
err1:
	free_pg_vec(pg_vec, order, block_nr);
	return NULL;
}

static int netlink_set_ring(struct sock *sk, struct nl_mmap_req *req,
			    bool closing, bool tx_ring)
{
	struct netlink_sock *nlk = nlk_sk(sk);
	struct netlink_ring *ring;
	struct sk_buff_head *queue;
	void **pg_vec = NULL;
	unsigned int order = 0;
	int err;

	ring  = tx_ring ? &nlk->tx_ring : &nlk->rx_ring;
	queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;

	if (!closing) {
		if (atomic_read(&nlk->mapped))
			return -EBUSY;
		if (atomic_read(&ring->pending))
			return -EBUSY;
	}

	if (req->nm_block_nr) {
		if (ring->pg_vec != NULL)
			return -EBUSY;

		if ((int)req->nm_block_size <= 0)
			return -EINVAL;
		if (!IS_ALIGNED(req->nm_block_size, PAGE_SIZE))
			return -EINVAL;
		if (req->nm_frame_size < NL_MMAP_HDRLEN)
			return -EINVAL;
		if (!IS_ALIGNED(req->nm_frame_size, NL_MMAP_MSG_ALIGNMENT))
			return -EINVAL;

		ring->frames_per_block = req->nm_block_size /
					 req->nm_frame_size;
		if (ring->frames_per_block == 0)
			return -EINVAL;
		if (ring->frames_per_block * req->nm_block_nr !=
		    req->nm_frame_nr)
			return -EINVAL;

		order = get_order(req->nm_block_size);
		pg_vec = alloc_pg_vec(nlk, req, order);
		if (pg_vec == NULL)
			return -ENOMEM;
	} else {
		if (req->nm_frame_nr)
			return -EINVAL;
	}

	err = -EBUSY;
	mutex_lock(&nlk->pg_vec_lock);
	if (closing || atomic_read(&nlk->mapped) == 0) {
		err = 0;
		spin_lock_bh(&queue->lock);

		ring->frame_max		= req->nm_frame_nr - 1;
		ring->head		= 0;
		ring->frame_size	= req->nm_frame_size;
		ring->pg_vec_pages	= req->nm_block_size / PAGE_SIZE;

		swap(ring->pg_vec_len, req->nm_block_nr);
		swap(ring->pg_vec_order, order);
		swap(ring->pg_vec, pg_vec);

		__skb_queue_purge(queue);
		spin_unlock_bh(&queue->lock);

		WARN_ON(atomic_read(&nlk->mapped));
	}
	mutex_unlock(&nlk->pg_vec_lock);

	if (pg_vec)
		free_pg_vec(pg_vec, order, req->nm_block_nr);
	return err;
}

static void netlink_mm_open(struct vm_area_struct *vma)
{
	struct file *file = vma->vm_file;
	struct socket *sock = file->private_data;
	struct sock *sk = sock->sk;

	if (sk)
		atomic_inc(&nlk_sk(sk)->mapped);
}

static void netlink_mm_close(struct vm_area_struct *vma)
{
	struct file *file = vma->vm_file;
	struct socket *sock = file->private_data;
	struct sock *sk = sock->sk;

	if (sk)
		atomic_dec(&nlk_sk(sk)->mapped);
}

static const struct vm_operations_struct netlink_mmap_ops = {
	.open	= netlink_mm_open,
	.close	= netlink_mm_close,
};

static int netlink_mmap(struct file *file, struct socket *sock,
			struct vm_area_struct *vma)
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
	struct netlink_ring *ring;
	unsigned long start, size, expected;
	unsigned int i;
	int err = -EINVAL;

	if (vma->vm_pgoff)
		return -EINVAL;

	mutex_lock(&nlk->pg_vec_lock);

	expected = 0;
	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {
		if (ring->pg_vec == NULL)
			continue;
		expected += ring->pg_vec_len * ring->pg_vec_pages * PAGE_SIZE;
	}

	if (expected == 0)
		goto out;

	size = vma->vm_end - vma->vm_start;
	if (size != expected)
		goto out;

	start = vma->vm_start;
	for (ring = &nlk->rx_ring; ring <= &nlk->tx_ring; ring++) {
		if (ring->pg_vec == NULL)
			continue;

		for (i = 0; i < ring->pg_vec_len; i++) {
			struct page *page;
			void *kaddr = ring->pg_vec[i];
			unsigned int pg_num;

			for (pg_num = 0; pg_num < ring->pg_vec_pages; pg_num++) {
				page = pgvec_to_page(kaddr);
				err = vm_insert_page(vma, start, page);
				if (err < 0)
					goto out;
				start += PAGE_SIZE;
				kaddr += PAGE_SIZE;
			}
		}
	}

	atomic_inc(&nlk->mapped);
	vma->vm_ops = &netlink_mmap_ops;
	err = 0;
out:
	mutex_unlock(&nlk->pg_vec_lock);
	return 0;
}
376 377 378 379 380 381 382 383

static void netlink_frame_flush_dcache(const struct nl_mmap_hdr *hdr)
{
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 1
	struct page *p_start, *p_end;

	/* First page is flushed through netlink_{get,set}_status */
	p_start = pgvec_to_page(hdr + PAGE_SIZE);
384
	p_end   = pgvec_to_page((void *)hdr + NL_MMAP_HDRLEN + hdr->nm_len - 1);
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
	while (p_start <= p_end) {
		flush_dcache_page(p_start);
		p_start++;
	}
#endif
}

static enum nl_mmap_status netlink_get_status(const struct nl_mmap_hdr *hdr)
{
	smp_rmb();
	flush_dcache_page(pgvec_to_page(hdr));
	return hdr->nm_status;
}

static void netlink_set_status(struct nl_mmap_hdr *hdr,
			       enum nl_mmap_status status)
{
	hdr->nm_status = status;
	flush_dcache_page(pgvec_to_page(hdr));
	smp_wmb();
}

static struct nl_mmap_hdr *
__netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos)
{
	unsigned int pg_vec_pos, frame_off;

	pg_vec_pos = pos / ring->frames_per_block;
	frame_off  = pos % ring->frames_per_block;

	return ring->pg_vec[pg_vec_pos] + (frame_off * ring->frame_size);
}

static struct nl_mmap_hdr *
netlink_lookup_frame(const struct netlink_ring *ring, unsigned int pos,
		     enum nl_mmap_status status)
{
	struct nl_mmap_hdr *hdr;

	hdr = __netlink_lookup_frame(ring, pos);
	if (netlink_get_status(hdr) != status)
		return NULL;

	return hdr;
}

static struct nl_mmap_hdr *
netlink_current_frame(const struct netlink_ring *ring,
		      enum nl_mmap_status status)
{
	return netlink_lookup_frame(ring, ring->head, status);
}

static struct nl_mmap_hdr *
netlink_previous_frame(const struct netlink_ring *ring,
		       enum nl_mmap_status status)
{
	unsigned int prev;

	prev = ring->head ? ring->head - 1 : ring->frame_max;
	return netlink_lookup_frame(ring, prev, status);
}

static void netlink_increment_head(struct netlink_ring *ring)
{
	ring->head = ring->head != ring->frame_max ? ring->head + 1 : 0;
}

static void netlink_forward_ring(struct netlink_ring *ring)
{
	unsigned int head = ring->head, pos = head;
	const struct nl_mmap_hdr *hdr;

	do {
		hdr = __netlink_lookup_frame(ring, pos);
		if (hdr->nm_status == NL_MMAP_STATUS_UNUSED)
			break;
		if (hdr->nm_status != NL_MMAP_STATUS_SKIP)
			break;
		netlink_increment_head(ring);
	} while (ring->head != head);
}

468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
static bool netlink_dump_space(struct netlink_sock *nlk)
{
	struct netlink_ring *ring = &nlk->rx_ring;
	struct nl_mmap_hdr *hdr;
	unsigned int n;

	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);
	if (hdr == NULL)
		return false;

	n = ring->head + ring->frame_max / 2;
	if (n > ring->frame_max)
		n -= ring->frame_max;

	hdr = __netlink_lookup_frame(ring, n);

	return hdr->nm_status == NL_MMAP_STATUS_UNUSED;
}

487 488 489 490 491 492
static unsigned int netlink_poll(struct file *file, struct socket *sock,
				 poll_table *wait)
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
	unsigned int mask;
493
	int err;
494

495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
	if (nlk->rx_ring.pg_vec != NULL) {
		/* Memory mapped sockets don't call recvmsg(), so flow control
		 * for dumps is performed here. A dump is allowed to continue
		 * if at least half the ring is unused.
		 */
		while (nlk->cb != NULL && netlink_dump_space(nlk)) {
			err = netlink_dump(sk);
			if (err < 0) {
				sk->sk_err = err;
				sk->sk_error_report(sk);
				break;
			}
		}
		netlink_rcv_wake(sk);
	}
510

511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
	mask = datagram_poll(file, sock, wait);

	spin_lock_bh(&sk->sk_receive_queue.lock);
	if (nlk->rx_ring.pg_vec) {
		netlink_forward_ring(&nlk->rx_ring);
		if (!netlink_previous_frame(&nlk->rx_ring, NL_MMAP_STATUS_UNUSED))
			mask |= POLLIN | POLLRDNORM;
	}
	spin_unlock_bh(&sk->sk_receive_queue.lock);

	spin_lock_bh(&sk->sk_write_queue.lock);
	if (nlk->tx_ring.pg_vec) {
		if (netlink_current_frame(&nlk->tx_ring, NL_MMAP_STATUS_UNUSED))
			mask |= POLLOUT | POLLWRNORM;
	}
	spin_unlock_bh(&sk->sk_write_queue.lock);

	return mask;
}

static struct nl_mmap_hdr *netlink_mmap_hdr(struct sk_buff *skb)
{
	return (struct nl_mmap_hdr *)(skb->head - NL_MMAP_HDRLEN);
}

static void netlink_ring_setup_skb(struct sk_buff *skb, struct sock *sk,
				   struct netlink_ring *ring,
				   struct nl_mmap_hdr *hdr)
{
	unsigned int size;
	void *data;

	size = ring->frame_size - NL_MMAP_HDRLEN;
	data = (void *)hdr + NL_MMAP_HDRLEN;

	skb->head	= data;
	skb->data	= data;
	skb_reset_tail_pointer(skb);
	skb->end	= skb->tail + size;
	skb->len	= 0;

	skb->destructor	= netlink_skb_destructor;
	NETLINK_CB(skb).flags |= NETLINK_SKB_MMAPED;
	NETLINK_CB(skb).sk = sk;
}
556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653

static int netlink_mmap_sendmsg(struct sock *sk, struct msghdr *msg,
				u32 dst_portid, u32 dst_group,
				struct sock_iocb *siocb)
{
	struct netlink_sock *nlk = nlk_sk(sk);
	struct netlink_ring *ring;
	struct nl_mmap_hdr *hdr;
	struct sk_buff *skb;
	unsigned int maxlen;
	bool excl = true;
	int err = 0, len = 0;

	/* Netlink messages are validated by the receiver before processing.
	 * In order to avoid userspace changing the contents of the message
	 * after validation, the socket and the ring may only be used by a
	 * single process, otherwise we fall back to copying.
	 */
	if (atomic_long_read(&sk->sk_socket->file->f_count) > 2 ||
	    atomic_read(&nlk->mapped) > 1)
		excl = false;

	mutex_lock(&nlk->pg_vec_lock);

	ring   = &nlk->tx_ring;
	maxlen = ring->frame_size - NL_MMAP_HDRLEN;

	do {
		hdr = netlink_current_frame(ring, NL_MMAP_STATUS_VALID);
		if (hdr == NULL) {
			if (!(msg->msg_flags & MSG_DONTWAIT) &&
			    atomic_read(&nlk->tx_ring.pending))
				schedule();
			continue;
		}
		if (hdr->nm_len > maxlen) {
			err = -EINVAL;
			goto out;
		}

		netlink_frame_flush_dcache(hdr);

		if (likely(dst_portid == 0 && dst_group == 0 && excl)) {
			skb = alloc_skb_head(GFP_KERNEL);
			if (skb == NULL) {
				err = -ENOBUFS;
				goto out;
			}
			sock_hold(sk);
			netlink_ring_setup_skb(skb, sk, ring, hdr);
			NETLINK_CB(skb).flags |= NETLINK_SKB_TX;
			__skb_put(skb, hdr->nm_len);
			netlink_set_status(hdr, NL_MMAP_STATUS_RESERVED);
			atomic_inc(&ring->pending);
		} else {
			skb = alloc_skb(hdr->nm_len, GFP_KERNEL);
			if (skb == NULL) {
				err = -ENOBUFS;
				goto out;
			}
			__skb_put(skb, hdr->nm_len);
			memcpy(skb->data, (void *)hdr + NL_MMAP_HDRLEN, hdr->nm_len);
			netlink_set_status(hdr, NL_MMAP_STATUS_UNUSED);
		}

		netlink_increment_head(ring);

		NETLINK_CB(skb).portid	  = nlk->portid;
		NETLINK_CB(skb).dst_group = dst_group;
		NETLINK_CB(skb).creds	  = siocb->scm->creds;

		err = security_netlink_send(sk, skb);
		if (err) {
			kfree_skb(skb);
			goto out;
		}

		if (unlikely(dst_group)) {
			atomic_inc(&skb->users);
			netlink_broadcast(sk, skb, dst_portid, dst_group,
					  GFP_KERNEL);
		}
		err = netlink_unicast(sk, skb, dst_portid,
				      msg->msg_flags & MSG_DONTWAIT);
		if (err < 0)
			goto out;
		len += err;

	} while (hdr != NULL ||
		 (!(msg->msg_flags & MSG_DONTWAIT) &&
		  atomic_read(&nlk->tx_ring.pending)));

	if (len > 0)
		err = len;
out:
	mutex_unlock(&nlk->pg_vec_lock);
	return err;
}
654 655 656 657 658 659 660 661 662

static void netlink_queue_mmaped_skb(struct sock *sk, struct sk_buff *skb)
{
	struct nl_mmap_hdr *hdr;

	hdr = netlink_mmap_hdr(skb);
	hdr->nm_len	= skb->len;
	hdr->nm_group	= NETLINK_CB(skb).dst_group;
	hdr->nm_pid	= NETLINK_CB(skb).creds.pid;
663 664
	hdr->nm_uid	= from_kuid(sk_user_ns(sk), NETLINK_CB(skb).creds.uid);
	hdr->nm_gid	= from_kgid(sk_user_ns(sk), NETLINK_CB(skb).creds.gid);
665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682
	netlink_frame_flush_dcache(hdr);
	netlink_set_status(hdr, NL_MMAP_STATUS_VALID);

	NETLINK_CB(skb).flags |= NETLINK_SKB_DELIVERED;
	kfree_skb(skb);
}

static void netlink_ring_set_copied(struct sock *sk, struct sk_buff *skb)
{
	struct netlink_sock *nlk = nlk_sk(sk);
	struct netlink_ring *ring = &nlk->rx_ring;
	struct nl_mmap_hdr *hdr;

	spin_lock_bh(&sk->sk_receive_queue.lock);
	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);
	if (hdr == NULL) {
		spin_unlock_bh(&sk->sk_receive_queue.lock);
		kfree_skb(skb);
683
		netlink_overrun(sk);
684 685 686 687 688 689 690 691 692
		return;
	}
	netlink_increment_head(ring);
	__skb_queue_tail(&sk->sk_receive_queue, skb);
	spin_unlock_bh(&sk->sk_receive_queue.lock);

	hdr->nm_len	= skb->len;
	hdr->nm_group	= NETLINK_CB(skb).dst_group;
	hdr->nm_pid	= NETLINK_CB(skb).creds.pid;
693 694
	hdr->nm_uid	= from_kuid(sk_user_ns(sk), NETLINK_CB(skb).creds.uid);
	hdr->nm_gid	= from_kgid(sk_user_ns(sk), NETLINK_CB(skb).creds.gid);
695 696 697
	netlink_set_status(hdr, NL_MMAP_STATUS_COPY);
}

698
#else /* CONFIG_NETLINK_MMAP */
699
#define netlink_skb_is_mmaped(skb)	false
700
#define netlink_rx_is_mmaped(sk)	false
701
#define netlink_tx_is_mmaped(sk)	false
702
#define netlink_mmap			sock_no_mmap
703
#define netlink_poll			datagram_poll
704
#define netlink_mmap_sendmsg(sk, msg, dst_portid, dst_group, siocb)	0
705 706
#endif /* CONFIG_NETLINK_MMAP */

E
Eric Dumazet 已提交
707 708 709 710 711 712
static void netlink_destroy_callback(struct netlink_callback *cb)
{
	kfree_skb(cb->skb);
	kfree(cb);
}

713 714 715 716 717 718
static void netlink_consume_callback(struct netlink_callback *cb)
{
	consume_skb(cb->skb);
	kfree(cb);
}

719 720
static void netlink_skb_destructor(struct sk_buff *skb)
{
721 722 723 724 725 726 727 728 729 730 731 732 733 734
#ifdef CONFIG_NETLINK_MMAP
	struct nl_mmap_hdr *hdr;
	struct netlink_ring *ring;
	struct sock *sk;

	/* If a packet from the kernel to userspace was freed because of an
	 * error without being delivered to userspace, the kernel must reset
	 * the status. In the direction userspace to kernel, the status is
	 * always reset here after the packet was processed and freed.
	 */
	if (netlink_skb_is_mmaped(skb)) {
		hdr = netlink_mmap_hdr(skb);
		sk = NETLINK_CB(skb).sk;

735 736 737 738 739 740 741 742 743
		if (NETLINK_CB(skb).flags & NETLINK_SKB_TX) {
			netlink_set_status(hdr, NL_MMAP_STATUS_UNUSED);
			ring = &nlk_sk(sk)->tx_ring;
		} else {
			if (!(NETLINK_CB(skb).flags & NETLINK_SKB_DELIVERED)) {
				hdr->nm_len = 0;
				netlink_set_status(hdr, NL_MMAP_STATUS_VALID);
			}
			ring = &nlk_sk(sk)->rx_ring;
744 745 746 747 748 749
		}

		WARN_ON(atomic_read(&ring->pending) == 0);
		atomic_dec(&ring->pending);
		sock_put(sk);

750
		skb->head = NULL;
751 752 753 754
	}
#endif
	if (skb->sk != NULL)
		sock_rfree(skb);
755 756 757 758 759 760 761 762 763 764 765
}

static void netlink_skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
{
	WARN_ON(skb->sk != NULL);
	skb->sk = sk;
	skb->destructor = netlink_skb_destructor;
	atomic_add(skb->truesize, &sk->sk_rmem_alloc);
	sk_mem_charge(sk, skb->truesize);
}

L
Linus Torvalds 已提交
766 767
static void netlink_sock_destruct(struct sock *sk)
{
768 769 770 771 772
	struct netlink_sock *nlk = nlk_sk(sk);

	if (nlk->cb) {
		if (nlk->cb->done)
			nlk->cb->done(nlk->cb);
773 774

		module_put(nlk->cb->module);
775 776 777
		netlink_destroy_callback(nlk->cb);
	}

L
Linus Torvalds 已提交
778
	skb_queue_purge(&sk->sk_receive_queue);
779 780 781 782 783 784 785 786 787 788 789 790
#ifdef CONFIG_NETLINK_MMAP
	if (1) {
		struct nl_mmap_req req;

		memset(&req, 0, sizeof(req));
		if (nlk->rx_ring.pg_vec)
			netlink_set_ring(sk, &req, true, false);
		memset(&req, 0, sizeof(req));
		if (nlk->tx_ring.pg_vec)
			netlink_set_ring(sk, &req, true, true);
	}
#endif /* CONFIG_NETLINK_MMAP */
L
Linus Torvalds 已提交
791 792

	if (!sock_flag(sk, SOCK_DEAD)) {
793
		printk(KERN_ERR "Freeing alive netlink socket %p\n", sk);
L
Linus Torvalds 已提交
794 795
		return;
	}
796 797 798 799

	WARN_ON(atomic_read(&sk->sk_rmem_alloc));
	WARN_ON(atomic_read(&sk->sk_wmem_alloc));
	WARN_ON(nlk_sk(sk)->groups);
L
Linus Torvalds 已提交
800 801
}

802 803
/* This lock without WQ_FLAG_EXCLUSIVE is good on UP and it is _very_ bad on
 * SMP. Look, when several writers sleep and reader wakes them up, all but one
L
Linus Torvalds 已提交
804 805 806 807
 * immediately hit write lock and grab all the cpus. Exclusive sleep solves
 * this, _but_ remember, it adds useless work on UP machines.
 */

808
void netlink_table_grab(void)
809
	__acquires(nl_table_lock)
L
Linus Torvalds 已提交
810
{
811 812
	might_sleep();

813
	write_lock_irq(&nl_table_lock);
L
Linus Torvalds 已提交
814 815 816 817 818

	if (atomic_read(&nl_table_users)) {
		DECLARE_WAITQUEUE(wait, current);

		add_wait_queue_exclusive(&nl_table_wait, &wait);
819
		for (;;) {
L
Linus Torvalds 已提交
820 821 822
			set_current_state(TASK_UNINTERRUPTIBLE);
			if (atomic_read(&nl_table_users) == 0)
				break;
823
			write_unlock_irq(&nl_table_lock);
L
Linus Torvalds 已提交
824
			schedule();
825
			write_lock_irq(&nl_table_lock);
L
Linus Torvalds 已提交
826 827 828 829 830 831 832
		}

		__set_current_state(TASK_RUNNING);
		remove_wait_queue(&nl_table_wait, &wait);
	}
}

833
void netlink_table_ungrab(void)
834
	__releases(nl_table_lock)
L
Linus Torvalds 已提交
835
{
836
	write_unlock_irq(&nl_table_lock);
L
Linus Torvalds 已提交
837 838 839
	wake_up(&nl_table_wait);
}

840
static inline void
L
Linus Torvalds 已提交
841 842 843 844 845 846 847 848 849
netlink_lock_table(void)
{
	/* read_lock() synchronizes us to netlink_table_grab */

	read_lock(&nl_table_lock);
	atomic_inc(&nl_table_users);
	read_unlock(&nl_table_lock);
}

850
static inline void
L
Linus Torvalds 已提交
851 852 853 854 855 856
netlink_unlock_table(void)
{
	if (atomic_dec_and_test(&nl_table_users))
		wake_up(&nl_table_wait);
}

857
static struct sock *netlink_lookup(struct net *net, int protocol, u32 portid)
L
Linus Torvalds 已提交
858
{
859
	struct nl_portid_hash *hash = &nl_table[protocol].hash;
L
Linus Torvalds 已提交
860 861 862 863
	struct hlist_head *head;
	struct sock *sk;

	read_lock(&nl_table_lock);
864
	head = nl_portid_hashfn(hash, portid);
865
	sk_for_each(sk, head) {
866
		if (net_eq(sock_net(sk), net) && (nlk_sk(sk)->portid == portid)) {
L
Linus Torvalds 已提交
867 868 869 870 871 872 873 874 875 876
			sock_hold(sk);
			goto found;
		}
	}
	sk = NULL;
found:
	read_unlock(&nl_table_lock);
	return sk;
}

877
static struct hlist_head *nl_portid_hash_zalloc(size_t size)
L
Linus Torvalds 已提交
878 879
{
	if (size <= PAGE_SIZE)
E
Eric Dumazet 已提交
880
		return kzalloc(size, GFP_ATOMIC);
L
Linus Torvalds 已提交
881 882
	else
		return (struct hlist_head *)
E
Eric Dumazet 已提交
883 884
			__get_free_pages(GFP_ATOMIC | __GFP_ZERO,
					 get_order(size));
L
Linus Torvalds 已提交
885 886
}

887
static void nl_portid_hash_free(struct hlist_head *table, size_t size)
L
Linus Torvalds 已提交
888 889 890 891 892 893 894
{
	if (size <= PAGE_SIZE)
		kfree(table);
	else
		free_pages((unsigned long)table, get_order(size));
}

895
static int nl_portid_hash_rehash(struct nl_portid_hash *hash, int grow)
L
Linus Torvalds 已提交
896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
{
	unsigned int omask, mask, shift;
	size_t osize, size;
	struct hlist_head *otable, *table;
	int i;

	omask = mask = hash->mask;
	osize = size = (mask + 1) * sizeof(*table);
	shift = hash->shift;

	if (grow) {
		if (++shift > hash->max_shift)
			return 0;
		mask = mask * 2 + 1;
		size *= 2;
	}

913
	table = nl_portid_hash_zalloc(size);
L
Linus Torvalds 已提交
914 915 916 917 918 919 920 921 922 923 924
	if (!table)
		return 0;

	otable = hash->table;
	hash->table = table;
	hash->mask = mask;
	hash->shift = shift;
	get_random_bytes(&hash->rnd, sizeof(hash->rnd));

	for (i = 0; i <= omask; i++) {
		struct sock *sk;
925
		struct hlist_node *tmp;
L
Linus Torvalds 已提交
926

927
		sk_for_each_safe(sk, tmp, &otable[i])
928
			__sk_add_node(sk, nl_portid_hashfn(hash, nlk_sk(sk)->portid));
L
Linus Torvalds 已提交
929 930
	}

931
	nl_portid_hash_free(otable, osize);
L
Linus Torvalds 已提交
932 933 934 935
	hash->rehash_time = jiffies + 10 * 60 * HZ;
	return 1;
}

936
static inline int nl_portid_hash_dilute(struct nl_portid_hash *hash, int len)
L
Linus Torvalds 已提交
937 938 939
{
	int avg = hash->entries >> hash->shift;

940
	if (unlikely(avg > 1) && nl_portid_hash_rehash(hash, 1))
L
Linus Torvalds 已提交
941 942 943
		return 1;

	if (unlikely(len > avg) && time_after(jiffies, hash->rehash_time)) {
944
		nl_portid_hash_rehash(hash, 0);
L
Linus Torvalds 已提交
945 946 947 948 949 950
		return 1;
	}

	return 0;
}

951
static const struct proto_ops netlink_ops;
L
Linus Torvalds 已提交
952

953 954 955 956 957 958
static void
netlink_update_listeners(struct sock *sk)
{
	struct netlink_table *tbl = &nl_table[sk->sk_protocol];
	unsigned long mask;
	unsigned int i;
959 960 961 962 963
	struct listeners *listeners;

	listeners = nl_deref_protected(tbl->listeners);
	if (!listeners)
		return;
964

965
	for (i = 0; i < NLGRPLONGS(tbl->groups); i++) {
966
		mask = 0;
967
		sk_for_each_bound(sk, &tbl->mc_list) {
968 969 970
			if (i < NLGRPLONGS(nlk_sk(sk)->ngroups))
				mask |= nlk_sk(sk)->groups[i];
		}
971
		listeners->masks[i] = mask;
972 973 974 975 976
	}
	/* this function is only called with the netlink table "grabbed", which
	 * makes sure updates are visible before bind or setsockopt return. */
}

977
static int netlink_insert(struct sock *sk, struct net *net, u32 portid)
L
Linus Torvalds 已提交
978
{
979
	struct nl_portid_hash *hash = &nl_table[sk->sk_protocol].hash;
L
Linus Torvalds 已提交
980 981 982 983 984 985
	struct hlist_head *head;
	int err = -EADDRINUSE;
	struct sock *osk;
	int len;

	netlink_table_grab();
986
	head = nl_portid_hashfn(hash, portid);
L
Linus Torvalds 已提交
987
	len = 0;
988
	sk_for_each(osk, head) {
989
		if (net_eq(sock_net(osk), net) && (nlk_sk(osk)->portid == portid))
L
Linus Torvalds 已提交
990 991 992
			break;
		len++;
	}
993
	if (osk)
L
Linus Torvalds 已提交
994 995 996
		goto err;

	err = -EBUSY;
997
	if (nlk_sk(sk)->portid)
L
Linus Torvalds 已提交
998 999 1000 1001 1002 1003
		goto err;

	err = -ENOMEM;
	if (BITS_PER_LONG > 32 && unlikely(hash->entries >= UINT_MAX))
		goto err;

1004 1005
	if (len && nl_portid_hash_dilute(hash, len))
		head = nl_portid_hashfn(hash, portid);
L
Linus Torvalds 已提交
1006
	hash->entries++;
1007
	nlk_sk(sk)->portid = portid;
L
Linus Torvalds 已提交
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
	sk_add_node(sk, head);
	err = 0;

err:
	netlink_table_ungrab();
	return err;
}

static void netlink_remove(struct sock *sk)
{
	netlink_table_grab();
1019 1020
	if (sk_del_node_init(sk))
		nl_table[sk->sk_protocol].hash.entries--;
1021
	if (nlk_sk(sk)->subscriptions)
L
Linus Torvalds 已提交
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
		__sk_del_bind_node(sk);
	netlink_table_ungrab();
}

static struct proto netlink_proto = {
	.name	  = "NETLINK",
	.owner	  = THIS_MODULE,
	.obj_size = sizeof(struct netlink_sock),
};

1032 1033
static int __netlink_create(struct net *net, struct socket *sock,
			    struct mutex *cb_mutex, int protocol)
L
Linus Torvalds 已提交
1034 1035 1036
{
	struct sock *sk;
	struct netlink_sock *nlk;
1037 1038 1039

	sock->ops = &netlink_ops;

1040
	sk = sk_alloc(net, PF_NETLINK, GFP_KERNEL, &netlink_proto);
1041 1042 1043 1044 1045 1046
	if (!sk)
		return -ENOMEM;

	sock_init_data(sock, sk);

	nlk = nlk_sk(sk);
E
Eric Dumazet 已提交
1047
	if (cb_mutex) {
1048
		nlk->cb_mutex = cb_mutex;
E
Eric Dumazet 已提交
1049
	} else {
1050 1051 1052
		nlk->cb_mutex = &nlk->cb_def_mutex;
		mutex_init(nlk->cb_mutex);
	}
1053
	init_waitqueue_head(&nlk->wait);
1054 1055 1056
#ifdef CONFIG_NETLINK_MMAP
	mutex_init(&nlk->pg_vec_lock);
#endif
1057 1058 1059 1060 1061 1062

	sk->sk_destruct = netlink_sock_destruct;
	sk->sk_protocol = protocol;
	return 0;
}

1063 1064
static int netlink_create(struct net *net, struct socket *sock, int protocol,
			  int kern)
1065 1066
{
	struct module *module = NULL;
1067
	struct mutex *cb_mutex;
1068
	struct netlink_sock *nlk;
1069
	void (*bind)(int group);
1070
	int err = 0;
L
Linus Torvalds 已提交
1071 1072 1073 1074 1075 1076

	sock->state = SS_UNCONNECTED;

	if (sock->type != SOCK_RAW && sock->type != SOCK_DGRAM)
		return -ESOCKTNOSUPPORT;

1077
	if (protocol < 0 || protocol >= MAX_LINKS)
L
Linus Torvalds 已提交
1078 1079
		return -EPROTONOSUPPORT;

1080
	netlink_lock_table();
1081
#ifdef CONFIG_MODULES
1082
	if (!nl_table[protocol].registered) {
1083
		netlink_unlock_table();
1084
		request_module("net-pf-%d-proto-%d", PF_NETLINK, protocol);
1085
		netlink_lock_table();
1086
	}
1087 1088 1089 1090
#endif
	if (nl_table[protocol].registered &&
	    try_module_get(nl_table[protocol].module))
		module = nl_table[protocol].module;
1091 1092
	else
		err = -EPROTONOSUPPORT;
1093
	cb_mutex = nl_table[protocol].cb_mutex;
1094
	bind = nl_table[protocol].bind;
1095
	netlink_unlock_table();
1096

1097 1098 1099
	if (err < 0)
		goto out;

1100 1101
	err = __netlink_create(net, sock, cb_mutex, protocol);
	if (err < 0)
1102 1103
		goto out_module;

1104
	local_bh_disable();
1105
	sock_prot_inuse_add(net, &netlink_proto, 1);
1106 1107
	local_bh_enable();

1108 1109
	nlk = nlk_sk(sock->sk);
	nlk->module = module;
1110
	nlk->netlink_bind = bind;
1111 1112
out:
	return err;
L
Linus Torvalds 已提交
1113

1114 1115 1116
out_module:
	module_put(module);
	goto out;
L
Linus Torvalds 已提交
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
}

static int netlink_release(struct socket *sock)
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk;

	if (!sk)
		return 0;

	netlink_remove(sk);
1128
	sock_orphan(sk);
L
Linus Torvalds 已提交
1129 1130
	nlk = nlk_sk(sk);

1131 1132 1133 1134
	/*
	 * OK. Socket is unlinked, any packets that arrive now
	 * will be purged.
	 */
L
Linus Torvalds 已提交
1135 1136 1137 1138 1139 1140

	sock->sk = NULL;
	wake_up_interruptible_all(&nlk->wait);

	skb_queue_purge(&sk->sk_write_queue);

1141
	if (nlk->portid) {
L
Linus Torvalds 已提交
1142
		struct netlink_notify n = {
1143
						.net = sock_net(sk),
L
Linus Torvalds 已提交
1144
						.protocol = sk->sk_protocol,
1145
						.portid = nlk->portid,
L
Linus Torvalds 已提交
1146
					  };
1147 1148
		atomic_notifier_call_chain(&netlink_chain,
				NETLINK_URELEASE, &n);
1149
	}
1150

1151
	module_put(nlk->module);
1152

1153
	netlink_table_grab();
1154
	if (netlink_is_kernel(sk)) {
1155 1156
		BUG_ON(nl_table[sk->sk_protocol].registered == 0);
		if (--nl_table[sk->sk_protocol].registered == 0) {
1157 1158 1159 1160 1161
			struct listeners *old;

			old = nl_deref_protected(nl_table[sk->sk_protocol].listeners);
			RCU_INIT_POINTER(nl_table[sk->sk_protocol].listeners, NULL);
			kfree_rcu(old, rcu);
1162
			nl_table[sk->sk_protocol].module = NULL;
1163 1164
			nl_table[sk->sk_protocol].bind = NULL;
			nl_table[sk->sk_protocol].flags = 0;
1165 1166
			nl_table[sk->sk_protocol].registered = 0;
		}
E
Eric Dumazet 已提交
1167
	} else if (nlk->subscriptions) {
1168
		netlink_update_listeners(sk);
E
Eric Dumazet 已提交
1169
	}
1170
	netlink_table_ungrab();
1171

1172 1173 1174
	kfree(nlk->groups);
	nlk->groups = NULL;

1175
	local_bh_disable();
1176
	sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1);
1177
	local_bh_enable();
L
Linus Torvalds 已提交
1178 1179 1180 1181 1182 1183 1184
	sock_put(sk);
	return 0;
}

static int netlink_autobind(struct socket *sock)
{
	struct sock *sk = sock->sk;
1185
	struct net *net = sock_net(sk);
1186
	struct nl_portid_hash *hash = &nl_table[sk->sk_protocol].hash;
L
Linus Torvalds 已提交
1187 1188
	struct hlist_head *head;
	struct sock *osk;
1189
	s32 portid = task_tgid_vnr(current);
L
Linus Torvalds 已提交
1190 1191 1192 1193 1194 1195
	int err;
	static s32 rover = -4097;

retry:
	cond_resched();
	netlink_table_grab();
1196
	head = nl_portid_hashfn(hash, portid);
1197
	sk_for_each(osk, head) {
1198
		if (!net_eq(sock_net(osk), net))
1199
			continue;
1200 1201 1202
		if (nlk_sk(osk)->portid == portid) {
			/* Bind collision, search negative portid values. */
			portid = rover--;
L
Linus Torvalds 已提交
1203 1204 1205 1206 1207 1208 1209 1210
			if (rover > -4097)
				rover = -4097;
			netlink_table_ungrab();
			goto retry;
		}
	}
	netlink_table_ungrab();

1211
	err = netlink_insert(sk, net, portid);
L
Linus Torvalds 已提交
1212 1213
	if (err == -EADDRINUSE)
		goto retry;
1214 1215 1216 1217 1218 1219

	/* If 2 threads race to autobind, that is fine.  */
	if (err == -EBUSY)
		err = 0;

	return err;
L
Linus Torvalds 已提交
1220 1221
}

1222
static inline int netlink_capable(const struct socket *sock, unsigned int flag)
1223
{
1224
	return (nl_table[sock->sk->sk_protocol].flags & flag) ||
1225
		ns_capable(sock_net(sock->sk)->user_ns, CAP_NET_ADMIN);
1226
}
L
Linus Torvalds 已提交
1227

1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239
static void
netlink_update_subscriptions(struct sock *sk, unsigned int subscriptions)
{
	struct netlink_sock *nlk = nlk_sk(sk);

	if (nlk->subscriptions && !subscriptions)
		__sk_del_bind_node(sk);
	else if (!nlk->subscriptions && subscriptions)
		sk_add_bind_node(sk, &nl_table[sk->sk_protocol].mc_list);
	nlk->subscriptions = subscriptions;
}

1240
static int netlink_realloc_groups(struct sock *sk)
1241 1242 1243
{
	struct netlink_sock *nlk = nlk_sk(sk);
	unsigned int groups;
1244
	unsigned long *new_groups;
1245 1246
	int err = 0;

1247 1248
	netlink_table_grab();

1249
	groups = nl_table[sk->sk_protocol].groups;
1250
	if (!nl_table[sk->sk_protocol].registered) {
1251
		err = -ENOENT;
1252 1253
		goto out_unlock;
	}
1254

1255 1256
	if (nlk->ngroups >= groups)
		goto out_unlock;
1257

1258 1259 1260 1261 1262
	new_groups = krealloc(nlk->groups, NLGRPSZ(groups), GFP_ATOMIC);
	if (new_groups == NULL) {
		err = -ENOMEM;
		goto out_unlock;
	}
1263
	memset((char *)new_groups + NLGRPSZ(nlk->ngroups), 0,
1264 1265 1266
	       NLGRPSZ(groups) - NLGRPSZ(nlk->ngroups));

	nlk->groups = new_groups;
1267
	nlk->ngroups = groups;
1268 1269 1270
 out_unlock:
	netlink_table_ungrab();
	return err;
1271 1272
}

1273 1274
static int netlink_bind(struct socket *sock, struct sockaddr *addr,
			int addr_len)
L
Linus Torvalds 已提交
1275 1276
{
	struct sock *sk = sock->sk;
1277
	struct net *net = sock_net(sk);
L
Linus Torvalds 已提交
1278 1279 1280
	struct netlink_sock *nlk = nlk_sk(sk);
	struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;
	int err;
1281

1282 1283 1284
	if (addr_len < sizeof(struct sockaddr_nl))
		return -EINVAL;

L
Linus Torvalds 已提交
1285 1286 1287 1288
	if (nladdr->nl_family != AF_NETLINK)
		return -EINVAL;

	/* Only superuser is allowed to listen multicasts */
1289
	if (nladdr->nl_groups) {
1290
		if (!netlink_capable(sock, NL_CFG_F_NONROOT_RECV))
1291
			return -EPERM;
1292 1293 1294
		err = netlink_realloc_groups(sk);
		if (err)
			return err;
1295
	}
L
Linus Torvalds 已提交
1296

1297 1298
	if (nlk->portid) {
		if (nladdr->nl_pid != nlk->portid)
L
Linus Torvalds 已提交
1299 1300 1301
			return -EINVAL;
	} else {
		err = nladdr->nl_pid ?
1302
			netlink_insert(sk, net, nladdr->nl_pid) :
L
Linus Torvalds 已提交
1303 1304 1305 1306 1307
			netlink_autobind(sock);
		if (err)
			return err;
	}

1308
	if (!nladdr->nl_groups && (nlk->groups == NULL || !(u32)nlk->groups[0]))
L
Linus Torvalds 已提交
1309 1310 1311
		return 0;

	netlink_table_grab();
1312
	netlink_update_subscriptions(sk, nlk->subscriptions +
1313 1314 1315
					 hweight32(nladdr->nl_groups) -
					 hweight32(nlk->groups[0]));
	nlk->groups[0] = (nlk->groups[0] & ~0xffffffffUL) | nladdr->nl_groups;
1316
	netlink_update_listeners(sk);
L
Linus Torvalds 已提交
1317 1318
	netlink_table_ungrab();

1319 1320 1321 1322 1323 1324 1325 1326 1327
	if (nlk->netlink_bind && nlk->groups[0]) {
		int i;

		for (i=0; i<nlk->ngroups; i++) {
			if (test_bit(i, nlk->groups))
				nlk->netlink_bind(i);
		}
	}

L
Linus Torvalds 已提交
1328 1329 1330 1331 1332 1333 1334 1335 1336
	return 0;
}

static int netlink_connect(struct socket *sock, struct sockaddr *addr,
			   int alen, int flags)
{
	int err = 0;
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
1337
	struct sockaddr_nl *nladdr = (struct sockaddr_nl *)addr;
L
Linus Torvalds 已提交
1338

1339 1340 1341
	if (alen < sizeof(addr->sa_family))
		return -EINVAL;

L
Linus Torvalds 已提交
1342 1343
	if (addr->sa_family == AF_UNSPEC) {
		sk->sk_state	= NETLINK_UNCONNECTED;
1344
		nlk->dst_portid	= 0;
1345
		nlk->dst_group  = 0;
L
Linus Torvalds 已提交
1346 1347 1348 1349 1350 1351
		return 0;
	}
	if (addr->sa_family != AF_NETLINK)
		return -EINVAL;

	/* Only superuser is allowed to send multicasts */
1352
	if (nladdr->nl_groups && !netlink_capable(sock, NL_CFG_F_NONROOT_SEND))
L
Linus Torvalds 已提交
1353 1354
		return -EPERM;

1355
	if (!nlk->portid)
L
Linus Torvalds 已提交
1356 1357 1358 1359
		err = netlink_autobind(sock);

	if (err == 0) {
		sk->sk_state	= NETLINK_CONNECTED;
1360
		nlk->dst_portid = nladdr->nl_pid;
1361
		nlk->dst_group  = ffs(nladdr->nl_groups);
L
Linus Torvalds 已提交
1362 1363 1364 1365 1366
	}

	return err;
}

1367 1368
static int netlink_getname(struct socket *sock, struct sockaddr *addr,
			   int *addr_len, int peer)
L
Linus Torvalds 已提交
1369 1370 1371
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
1372
	DECLARE_SOCKADDR(struct sockaddr_nl *, nladdr, addr);
1373

L
Linus Torvalds 已提交
1374 1375 1376 1377 1378
	nladdr->nl_family = AF_NETLINK;
	nladdr->nl_pad = 0;
	*addr_len = sizeof(*nladdr);

	if (peer) {
1379
		nladdr->nl_pid = nlk->dst_portid;
1380
		nladdr->nl_groups = netlink_group_mask(nlk->dst_group);
L
Linus Torvalds 已提交
1381
	} else {
1382
		nladdr->nl_pid = nlk->portid;
1383
		nladdr->nl_groups = nlk->groups ? nlk->groups[0] : 0;
L
Linus Torvalds 已提交
1384 1385 1386 1387
	}
	return 0;
}

1388
static struct sock *netlink_getsockbyportid(struct sock *ssk, u32 portid)
L
Linus Torvalds 已提交
1389 1390 1391 1392
{
	struct sock *sock;
	struct netlink_sock *nlk;

1393
	sock = netlink_lookup(sock_net(ssk), ssk->sk_protocol, portid);
L
Linus Torvalds 已提交
1394 1395 1396 1397 1398
	if (!sock)
		return ERR_PTR(-ECONNREFUSED);

	/* Don't bother queuing skb if kernel socket has no input function */
	nlk = nlk_sk(sock);
1399
	if (sock->sk_state == NETLINK_CONNECTED &&
1400
	    nlk->dst_portid != nlk_sk(ssk)->portid) {
L
Linus Torvalds 已提交
1401 1402 1403 1404 1405 1406 1407 1408
		sock_put(sock);
		return ERR_PTR(-ECONNREFUSED);
	}
	return sock;
}

struct sock *netlink_getsockbyfilp(struct file *filp)
{
A
Al Viro 已提交
1409
	struct inode *inode = file_inode(filp);
L
Linus Torvalds 已提交
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
	struct sock *sock;

	if (!S_ISSOCK(inode->i_mode))
		return ERR_PTR(-ENOTSOCK);

	sock = SOCKET_I(inode)->sk;
	if (sock->sk_family != AF_NETLINK)
		return ERR_PTR(-EINVAL);

	sock_hold(sock);
	return sock;
}

/*
 * Attach a skb to a netlink socket.
 * The caller must hold a reference to the destination socket. On error, the
 * reference is dropped. The skb is not send to the destination, just all
 * all error checks are performed and memory in the queue is reserved.
 * Return values:
 * < 0: error. skb freed, reference to sock dropped.
 * 0: continue
 * 1: repeat lookup - reference dropped while waiting for socket memory.
 */
1433
int netlink_attachskb(struct sock *sk, struct sk_buff *skb,
P
Patrick McHardy 已提交
1434
		      long *timeo, struct sock *ssk)
L
Linus Torvalds 已提交
1435 1436 1437 1438 1439
{
	struct netlink_sock *nlk;

	nlk = nlk_sk(sk);

1440 1441 1442
	if ((atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
	     test_bit(NETLINK_CONGESTED, &nlk->state)) &&
	    !netlink_skb_is_mmaped(skb)) {
L
Linus Torvalds 已提交
1443
		DECLARE_WAITQUEUE(wait, current);
P
Patrick McHardy 已提交
1444
		if (!*timeo) {
1445
			if (!ssk || netlink_is_kernel(ssk))
L
Linus Torvalds 已提交
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455
				netlink_overrun(sk);
			sock_put(sk);
			kfree_skb(skb);
			return -EAGAIN;
		}

		__set_current_state(TASK_INTERRUPTIBLE);
		add_wait_queue(&nlk->wait, &wait);

		if ((atomic_read(&sk->sk_rmem_alloc) > sk->sk_rcvbuf ||
1456
		     test_bit(NETLINK_CONGESTED, &nlk->state)) &&
L
Linus Torvalds 已提交
1457
		    !sock_flag(sk, SOCK_DEAD))
P
Patrick McHardy 已提交
1458
			*timeo = schedule_timeout(*timeo);
L
Linus Torvalds 已提交
1459 1460 1461 1462 1463 1464 1465

		__set_current_state(TASK_RUNNING);
		remove_wait_queue(&nlk->wait, &wait);
		sock_put(sk);

		if (signal_pending(current)) {
			kfree_skb(skb);
P
Patrick McHardy 已提交
1466
			return sock_intr_errno(*timeo);
L
Linus Torvalds 已提交
1467 1468 1469
		}
		return 1;
	}
1470
	netlink_skb_set_owner_r(skb, sk);
L
Linus Torvalds 已提交
1471 1472 1473
	return 0;
}

1474
static int __netlink_sendskb(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
1475 1476 1477
{
	int len = skb->len;

1478 1479 1480 1481 1482 1483 1484 1485
#ifdef CONFIG_NETLINK_MMAP
	if (netlink_skb_is_mmaped(skb))
		netlink_queue_mmaped_skb(sk, skb);
	else if (netlink_rx_is_mmaped(sk))
		netlink_ring_set_copied(sk, skb);
	else
#endif /* CONFIG_NETLINK_MMAP */
		skb_queue_tail(&sk->sk_receive_queue, skb);
L
Linus Torvalds 已提交
1486
	sk->sk_data_ready(sk, len);
1487 1488 1489 1490 1491 1492 1493
	return len;
}

int netlink_sendskb(struct sock *sk, struct sk_buff *skb)
{
	int len = __netlink_sendskb(sk, skb);

L
Linus Torvalds 已提交
1494 1495 1496 1497 1498 1499 1500 1501 1502 1503
	sock_put(sk);
	return len;
}

void netlink_detachskb(struct sock *sk, struct sk_buff *skb)
{
	kfree_skb(skb);
	sock_put(sk);
}

1504
static struct sk_buff *netlink_trim(struct sk_buff *skb, gfp_t allocation)
L
Linus Torvalds 已提交
1505 1506 1507
{
	int delta;

1508
	WARN_ON(skb->sk != NULL);
1509 1510
	if (netlink_skb_is_mmaped(skb))
		return skb;
L
Linus Torvalds 已提交
1511

1512
	delta = skb->end - skb->tail;
L
Linus Torvalds 已提交
1513 1514 1515 1516 1517 1518 1519
	if (delta * 2 < skb->truesize)
		return skb;

	if (skb_shared(skb)) {
		struct sk_buff *nskb = skb_clone(skb, allocation);
		if (!nskb)
			return skb;
1520
		consume_skb(skb);
L
Linus Torvalds 已提交
1521 1522 1523 1524 1525 1526 1527 1528 1529
		skb = nskb;
	}

	if (!pskb_expand_head(skb, 0, -delta, allocation))
		skb->truesize -= delta;

	return skb;
}

1530 1531
static int netlink_unicast_kernel(struct sock *sk, struct sk_buff *skb,
				  struct sock *ssk)
1532 1533 1534 1535 1536 1537 1538
{
	int ret;
	struct netlink_sock *nlk = nlk_sk(sk);

	ret = -ECONNREFUSED;
	if (nlk->netlink_rcv != NULL) {
		ret = skb->len;
1539
		netlink_skb_set_owner_r(skb, sk);
1540
		NETLINK_CB(skb).sk = ssk;
1541
		nlk->netlink_rcv(skb);
1542 1543 1544
		consume_skb(skb);
	} else {
		kfree_skb(skb);
1545 1546 1547 1548 1549 1550
	}
	sock_put(sk);
	return ret;
}

int netlink_unicast(struct sock *ssk, struct sk_buff *skb,
1551
		    u32 portid, int nonblock)
L
Linus Torvalds 已提交
1552 1553 1554 1555 1556 1557 1558 1559 1560
{
	struct sock *sk;
	int err;
	long timeo;

	skb = netlink_trim(skb, gfp_any());

	timeo = sock_sndtimeo(ssk, nonblock);
retry:
1561
	sk = netlink_getsockbyportid(ssk, portid);
L
Linus Torvalds 已提交
1562 1563 1564 1565
	if (IS_ERR(sk)) {
		kfree_skb(skb);
		return PTR_ERR(sk);
	}
1566
	if (netlink_is_kernel(sk))
1567
		return netlink_unicast_kernel(sk, skb, ssk);
1568

1569
	if (sk_filter(sk, skb)) {
W
Wang Chen 已提交
1570
		err = skb->len;
1571 1572 1573 1574 1575
		kfree_skb(skb);
		sock_put(sk);
		return err;
	}

1576
	err = netlink_attachskb(sk, skb, &timeo, ssk);
L
Linus Torvalds 已提交
1577 1578 1579 1580 1581
	if (err == 1)
		goto retry;
	if (err)
		return err;

1582
	return netlink_sendskb(sk, skb);
L
Linus Torvalds 已提交
1583
}
1584
EXPORT_SYMBOL(netlink_unicast);
L
Linus Torvalds 已提交
1585

1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632
struct sk_buff *netlink_alloc_skb(struct sock *ssk, unsigned int size,
				  u32 dst_portid, gfp_t gfp_mask)
{
#ifdef CONFIG_NETLINK_MMAP
	struct sock *sk = NULL;
	struct sk_buff *skb;
	struct netlink_ring *ring;
	struct nl_mmap_hdr *hdr;
	unsigned int maxlen;

	sk = netlink_getsockbyportid(ssk, dst_portid);
	if (IS_ERR(sk))
		goto out;

	ring = &nlk_sk(sk)->rx_ring;
	/* fast-path without atomic ops for common case: non-mmaped receiver */
	if (ring->pg_vec == NULL)
		goto out_put;

	skb = alloc_skb_head(gfp_mask);
	if (skb == NULL)
		goto err1;

	spin_lock_bh(&sk->sk_receive_queue.lock);
	/* check again under lock */
	if (ring->pg_vec == NULL)
		goto out_free;

	maxlen = ring->frame_size - NL_MMAP_HDRLEN;
	if (maxlen < size)
		goto out_free;

	netlink_forward_ring(ring);
	hdr = netlink_current_frame(ring, NL_MMAP_STATUS_UNUSED);
	if (hdr == NULL)
		goto err2;
	netlink_ring_setup_skb(skb, sk, ring, hdr);
	netlink_set_status(hdr, NL_MMAP_STATUS_RESERVED);
	atomic_inc(&ring->pending);
	netlink_increment_head(ring);

	spin_unlock_bh(&sk->sk_receive_queue.lock);
	return skb;

err2:
	kfree_skb(skb);
	spin_unlock_bh(&sk->sk_receive_queue.lock);
1633
	netlink_overrun(sk);
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648
err1:
	sock_put(sk);
	return NULL;

out_free:
	kfree_skb(skb);
	spin_unlock_bh(&sk->sk_receive_queue.lock);
out_put:
	sock_put(sk);
out:
#endif
	return alloc_skb(size, gfp_mask);
}
EXPORT_SYMBOL_GPL(netlink_alloc_skb);

1649 1650 1651
int netlink_has_listeners(struct sock *sk, unsigned int group)
{
	int res = 0;
1652
	struct listeners *listeners;
1653

1654
	BUG_ON(!netlink_is_kernel(sk));
1655 1656 1657 1658

	rcu_read_lock();
	listeners = rcu_dereference(nl_table[sk->sk_protocol].listeners);

1659
	if (listeners && group - 1 < nl_table[sk->sk_protocol].groups)
1660
		res = test_bit(group - 1, listeners->masks);
1661 1662 1663

	rcu_read_unlock();

1664 1665 1666 1667
	return res;
}
EXPORT_SYMBOL_GPL(netlink_has_listeners);

1668
static int netlink_broadcast_deliver(struct sock *sk, struct sk_buff *skb)
L
Linus Torvalds 已提交
1669 1670 1671 1672
{
	struct netlink_sock *nlk = nlk_sk(sk);

	if (atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf &&
1673
	    !test_bit(NETLINK_CONGESTED, &nlk->state)) {
1674
		netlink_skb_set_owner_r(skb, sk);
1675
		__netlink_sendskb(sk, skb);
1676
		return atomic_read(&sk->sk_rmem_alloc) > (sk->sk_rcvbuf >> 1);
L
Linus Torvalds 已提交
1677 1678 1679 1680 1681 1682
	}
	return -1;
}

struct netlink_broadcast_data {
	struct sock *exclude_sk;
1683
	struct net *net;
1684
	u32 portid;
L
Linus Torvalds 已提交
1685 1686
	u32 group;
	int failure;
1687
	int delivery_failure;
L
Linus Torvalds 已提交
1688 1689
	int congested;
	int delivered;
A
Al Viro 已提交
1690
	gfp_t allocation;
L
Linus Torvalds 已提交
1691
	struct sk_buff *skb, *skb2;
1692 1693
	int (*tx_filter)(struct sock *dsk, struct sk_buff *skb, void *data);
	void *tx_data;
L
Linus Torvalds 已提交
1694 1695
};

1696
static int do_one_broadcast(struct sock *sk,
L
Linus Torvalds 已提交
1697 1698 1699 1700 1701 1702 1703 1704
				   struct netlink_broadcast_data *p)
{
	struct netlink_sock *nlk = nlk_sk(sk);
	int val;

	if (p->exclude_sk == sk)
		goto out;

1705
	if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
1706
	    !test_bit(p->group - 1, nlk->groups))
L
Linus Torvalds 已提交
1707 1708
		goto out;

1709
	if (!net_eq(sock_net(sk), p->net))
1710 1711
		goto out;

L
Linus Torvalds 已提交
1712 1713 1714 1715 1716 1717 1718
	if (p->failure) {
		netlink_overrun(sk);
		goto out;
	}

	sock_hold(sk);
	if (p->skb2 == NULL) {
1719
		if (skb_shared(p->skb)) {
L
Linus Torvalds 已提交
1720 1721
			p->skb2 = skb_clone(p->skb, p->allocation);
		} else {
1722 1723 1724 1725 1726 1727
			p->skb2 = skb_get(p->skb);
			/*
			 * skb ownership may have been set when
			 * delivered to a previous socket.
			 */
			skb_orphan(p->skb2);
L
Linus Torvalds 已提交
1728 1729 1730 1731 1732 1733
		}
	}
	if (p->skb2 == NULL) {
		netlink_overrun(sk);
		/* Clone failed. Notify ALL listeners. */
		p->failure = 1;
1734 1735
		if (nlk->flags & NETLINK_BROADCAST_SEND_ERROR)
			p->delivery_failure = 1;
1736 1737 1738
	} else if (p->tx_filter && p->tx_filter(sk, p->skb2, p->tx_data)) {
		kfree_skb(p->skb2);
		p->skb2 = NULL;
1739 1740 1741
	} else if (sk_filter(sk, p->skb2)) {
		kfree_skb(p->skb2);
		p->skb2 = NULL;
L
Linus Torvalds 已提交
1742 1743
	} else if ((val = netlink_broadcast_deliver(sk, p->skb2)) < 0) {
		netlink_overrun(sk);
1744 1745
		if (nlk->flags & NETLINK_BROADCAST_SEND_ERROR)
			p->delivery_failure = 1;
L
Linus Torvalds 已提交
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
	} else {
		p->congested |= val;
		p->delivered = 1;
		p->skb2 = NULL;
	}
	sock_put(sk);

out:
	return 0;
}

1757
int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb, u32 portid,
1758 1759 1760
	u32 group, gfp_t allocation,
	int (*filter)(struct sock *dsk, struct sk_buff *skb, void *data),
	void *filter_data)
L
Linus Torvalds 已提交
1761
{
1762
	struct net *net = sock_net(ssk);
L
Linus Torvalds 已提交
1763 1764 1765 1766 1767 1768
	struct netlink_broadcast_data info;
	struct sock *sk;

	skb = netlink_trim(skb, allocation);

	info.exclude_sk = ssk;
1769
	info.net = net;
1770
	info.portid = portid;
L
Linus Torvalds 已提交
1771 1772
	info.group = group;
	info.failure = 0;
1773
	info.delivery_failure = 0;
L
Linus Torvalds 已提交
1774 1775 1776 1777 1778
	info.congested = 0;
	info.delivered = 0;
	info.allocation = allocation;
	info.skb = skb;
	info.skb2 = NULL;
1779 1780
	info.tx_filter = filter;
	info.tx_data = filter_data;
L
Linus Torvalds 已提交
1781 1782 1783 1784 1785

	/* While we sleep in clone, do not allow to change socket list */

	netlink_lock_table();

1786
	sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
L
Linus Torvalds 已提交
1787 1788
		do_one_broadcast(sk, &info);

1789
	consume_skb(skb);
1790

L
Linus Torvalds 已提交
1791 1792
	netlink_unlock_table();

1793 1794
	if (info.delivery_failure) {
		kfree_skb(info.skb2);
1795
		return -ENOBUFS;
E
Eric Dumazet 已提交
1796 1797
	}
	consume_skb(info.skb2);
1798

L
Linus Torvalds 已提交
1799 1800 1801 1802 1803 1804 1805
	if (info.delivered) {
		if (info.congested && (allocation & __GFP_WAIT))
			yield();
		return 0;
	}
	return -ESRCH;
}
1806 1807
EXPORT_SYMBOL(netlink_broadcast_filtered);

1808
int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, u32 portid,
1809 1810
		      u32 group, gfp_t allocation)
{
1811
	return netlink_broadcast_filtered(ssk, skb, portid, group, allocation,
1812 1813
		NULL, NULL);
}
1814
EXPORT_SYMBOL(netlink_broadcast);
L
Linus Torvalds 已提交
1815 1816 1817

struct netlink_set_err_data {
	struct sock *exclude_sk;
1818
	u32 portid;
L
Linus Torvalds 已提交
1819 1820 1821 1822
	u32 group;
	int code;
};

1823
static int do_one_set_err(struct sock *sk, struct netlink_set_err_data *p)
L
Linus Torvalds 已提交
1824 1825
{
	struct netlink_sock *nlk = nlk_sk(sk);
1826
	int ret = 0;
L
Linus Torvalds 已提交
1827 1828 1829 1830

	if (sk == p->exclude_sk)
		goto out;

O
Octavian Purdila 已提交
1831
	if (!net_eq(sock_net(sk), sock_net(p->exclude_sk)))
1832 1833
		goto out;

1834
	if (nlk->portid == p->portid || p->group - 1 >= nlk->ngroups ||
1835
	    !test_bit(p->group - 1, nlk->groups))
L
Linus Torvalds 已提交
1836 1837
		goto out;

1838 1839 1840 1841 1842
	if (p->code == ENOBUFS && nlk->flags & NETLINK_RECV_NO_ENOBUFS) {
		ret = 1;
		goto out;
	}

L
Linus Torvalds 已提交
1843 1844 1845
	sk->sk_err = p->code;
	sk->sk_error_report(sk);
out:
1846
	return ret;
L
Linus Torvalds 已提交
1847 1848
}

1849 1850 1851
/**
 * netlink_set_err - report error to broadcast listeners
 * @ssk: the kernel netlink socket, as returned by netlink_kernel_create()
1852
 * @portid: the PORTID of a process that we want to skip (if any)
1853 1854
 * @groups: the broadcast group that will notice the error
 * @code: error code, must be negative (as usual in kernelspace)
1855 1856 1857
 *
 * This function returns the number of broadcast listeners that have set the
 * NETLINK_RECV_NO_ENOBUFS socket option.
1858
 */
1859
int netlink_set_err(struct sock *ssk, u32 portid, u32 group, int code)
L
Linus Torvalds 已提交
1860 1861 1862
{
	struct netlink_set_err_data info;
	struct sock *sk;
1863
	int ret = 0;
L
Linus Torvalds 已提交
1864 1865

	info.exclude_sk = ssk;
1866
	info.portid = portid;
L
Linus Torvalds 已提交
1867
	info.group = group;
1868 1869
	/* sk->sk_err wants a positive error value */
	info.code = -code;
L
Linus Torvalds 已提交
1870 1871 1872

	read_lock(&nl_table_lock);

1873
	sk_for_each_bound(sk, &nl_table[ssk->sk_protocol].mc_list)
1874
		ret += do_one_set_err(sk, &info);
L
Linus Torvalds 已提交
1875 1876

	read_unlock(&nl_table_lock);
1877
	return ret;
L
Linus Torvalds 已提交
1878
}
1879
EXPORT_SYMBOL(netlink_set_err);
L
Linus Torvalds 已提交
1880

1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897
/* must be called with netlink table grabbed */
static void netlink_update_socket_mc(struct netlink_sock *nlk,
				     unsigned int group,
				     int is_new)
{
	int old, new = !!is_new, subscriptions;

	old = test_bit(group - 1, nlk->groups);
	subscriptions = nlk->subscriptions - old + new;
	if (new)
		__set_bit(group - 1, nlk->groups);
	else
		__clear_bit(group - 1, nlk->groups);
	netlink_update_subscriptions(&nlk->sk, subscriptions);
	netlink_update_listeners(&nlk->sk);
}

1898
static int netlink_setsockopt(struct socket *sock, int level, int optname,
1899
			      char __user *optval, unsigned int optlen)
1900 1901 1902
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
1903 1904
	unsigned int val = 0;
	int err;
1905 1906 1907 1908

	if (level != SOL_NETLINK)
		return -ENOPROTOOPT;

1909 1910
	if (optname != NETLINK_RX_RING && optname != NETLINK_TX_RING &&
	    optlen >= sizeof(int) &&
1911
	    get_user(val, (unsigned int __user *)optval))
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923
		return -EFAULT;

	switch (optname) {
	case NETLINK_PKTINFO:
		if (val)
			nlk->flags |= NETLINK_RECV_PKTINFO;
		else
			nlk->flags &= ~NETLINK_RECV_PKTINFO;
		err = 0;
		break;
	case NETLINK_ADD_MEMBERSHIP:
	case NETLINK_DROP_MEMBERSHIP: {
1924
		if (!netlink_capable(sock, NL_CFG_F_NONROOT_RECV))
1925
			return -EPERM;
1926 1927 1928
		err = netlink_realloc_groups(sk);
		if (err)
			return err;
1929 1930 1931
		if (!val || val - 1 >= nlk->ngroups)
			return -EINVAL;
		netlink_table_grab();
1932 1933
		netlink_update_socket_mc(nlk, val,
					 optname == NETLINK_ADD_MEMBERSHIP);
1934
		netlink_table_ungrab();
1935 1936 1937 1938

		if (nlk->netlink_bind)
			nlk->netlink_bind(val);

1939 1940 1941
		err = 0;
		break;
	}
1942 1943 1944 1945 1946 1947 1948
	case NETLINK_BROADCAST_ERROR:
		if (val)
			nlk->flags |= NETLINK_BROADCAST_SEND_ERROR;
		else
			nlk->flags &= ~NETLINK_BROADCAST_SEND_ERROR;
		err = 0;
		break;
1949 1950 1951
	case NETLINK_NO_ENOBUFS:
		if (val) {
			nlk->flags |= NETLINK_RECV_NO_ENOBUFS;
1952
			clear_bit(NETLINK_CONGESTED, &nlk->state);
1953
			wake_up_interruptible(&nlk->wait);
E
Eric Dumazet 已提交
1954
		} else {
1955
			nlk->flags &= ~NETLINK_RECV_NO_ENOBUFS;
E
Eric Dumazet 已提交
1956
		}
1957 1958
		err = 0;
		break;
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
#ifdef CONFIG_NETLINK_MMAP
	case NETLINK_RX_RING:
	case NETLINK_TX_RING: {
		struct nl_mmap_req req;

		/* Rings might consume more memory than queue limits, require
		 * CAP_NET_ADMIN.
		 */
		if (!capable(CAP_NET_ADMIN))
			return -EPERM;
		if (optlen < sizeof(req))
			return -EINVAL;
		if (copy_from_user(&req, optval, sizeof(req)))
			return -EFAULT;
		err = netlink_set_ring(sk, &req, false,
				       optname == NETLINK_TX_RING);
		break;
	}
#endif /* CONFIG_NETLINK_MMAP */
1978 1979 1980 1981 1982 1983 1984
	default:
		err = -ENOPROTOOPT;
	}
	return err;
}

static int netlink_getsockopt(struct socket *sock, int level, int optname,
1985
			      char __user *optval, int __user *optlen)
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004
{
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
	int len, val, err;

	if (level != SOL_NETLINK)
		return -ENOPROTOOPT;

	if (get_user(len, optlen))
		return -EFAULT;
	if (len < 0)
		return -EINVAL;

	switch (optname) {
	case NETLINK_PKTINFO:
		if (len < sizeof(int))
			return -EINVAL;
		len = sizeof(int);
		val = nlk->flags & NETLINK_RECV_PKTINFO ? 1 : 0;
H
Heiko Carstens 已提交
2005 2006 2007
		if (put_user(len, optlen) ||
		    put_user(val, optval))
			return -EFAULT;
2008 2009
		err = 0;
		break;
2010 2011 2012 2013 2014 2015 2016 2017 2018 2019
	case NETLINK_BROADCAST_ERROR:
		if (len < sizeof(int))
			return -EINVAL;
		len = sizeof(int);
		val = nlk->flags & NETLINK_BROADCAST_SEND_ERROR ? 1 : 0;
		if (put_user(len, optlen) ||
		    put_user(val, optval))
			return -EFAULT;
		err = 0;
		break;
2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
	case NETLINK_NO_ENOBUFS:
		if (len < sizeof(int))
			return -EINVAL;
		len = sizeof(int);
		val = nlk->flags & NETLINK_RECV_NO_ENOBUFS ? 1 : 0;
		if (put_user(len, optlen) ||
		    put_user(val, optval))
			return -EFAULT;
		err = 0;
		break;
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
	default:
		err = -ENOPROTOOPT;
	}
	return err;
}

static void netlink_cmsg_recv_pktinfo(struct msghdr *msg, struct sk_buff *skb)
{
	struct nl_pktinfo info;

	info.group = NETLINK_CB(skb).dst_group;
	put_cmsg(msg, SOL_NETLINK, NETLINK_PKTINFO, sizeof(info), &info);
}

L
Linus Torvalds 已提交
2044 2045 2046 2047 2048 2049
static int netlink_sendmsg(struct kiocb *kiocb, struct socket *sock,
			   struct msghdr *msg, size_t len)
{
	struct sock_iocb *siocb = kiocb_to_siocb(kiocb);
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
2050
	struct sockaddr_nl *addr = msg->msg_name;
2051
	u32 dst_portid;
2052
	u32 dst_group;
L
Linus Torvalds 已提交
2053 2054 2055 2056 2057 2058 2059
	struct sk_buff *skb;
	int err;
	struct scm_cookie scm;

	if (msg->msg_flags&MSG_OOB)
		return -EOPNOTSUPP;

2060
	if (NULL == siocb->scm)
L
Linus Torvalds 已提交
2061
		siocb->scm = &scm;
2062

2063
	err = scm_send(sock, msg, siocb->scm, true);
L
Linus Torvalds 已提交
2064 2065 2066 2067
	if (err < 0)
		return err;

	if (msg->msg_namelen) {
2068
		err = -EINVAL;
L
Linus Torvalds 已提交
2069
		if (addr->nl_family != AF_NETLINK)
2070
			goto out;
2071
		dst_portid = addr->nl_pid;
2072
		dst_group = ffs(addr->nl_groups);
2073
		err =  -EPERM;
2074
		if ((dst_group || dst_portid) &&
2075
		    !netlink_capable(sock, NL_CFG_F_NONROOT_SEND))
2076
			goto out;
L
Linus Torvalds 已提交
2077
	} else {
2078
		dst_portid = nlk->dst_portid;
2079
		dst_group = nlk->dst_group;
L
Linus Torvalds 已提交
2080 2081
	}

2082
	if (!nlk->portid) {
L
Linus Torvalds 已提交
2083 2084 2085 2086 2087
		err = netlink_autobind(sock);
		if (err)
			goto out;
	}

2088 2089 2090 2091 2092 2093 2094
	if (netlink_tx_is_mmaped(sk) &&
	    msg->msg_iov->iov_base == NULL) {
		err = netlink_mmap_sendmsg(sk, msg, dst_portid, dst_group,
					   siocb);
		goto out;
	}

L
Linus Torvalds 已提交
2095 2096 2097 2098
	err = -EMSGSIZE;
	if (len > sk->sk_sndbuf - 32)
		goto out;
	err = -ENOBUFS;
2099
	skb = alloc_skb(len, GFP_KERNEL);
2100
	if (skb == NULL)
L
Linus Torvalds 已提交
2101 2102
		goto out;

2103
	NETLINK_CB(skb).portid	= nlk->portid;
2104
	NETLINK_CB(skb).dst_group = dst_group;
2105
	NETLINK_CB(skb).creds	= siocb->scm->creds;
L
Linus Torvalds 已提交
2106 2107

	err = -EFAULT;
2108
	if (memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len)) {
L
Linus Torvalds 已提交
2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
		kfree_skb(skb);
		goto out;
	}

	err = security_netlink_send(sk, skb);
	if (err) {
		kfree_skb(skb);
		goto out;
	}

2119
	if (dst_group) {
L
Linus Torvalds 已提交
2120
		atomic_inc(&skb->users);
2121
		netlink_broadcast(sk, skb, dst_portid, dst_group, GFP_KERNEL);
L
Linus Torvalds 已提交
2122
	}
2123
	err = netlink_unicast(sk, skb, dst_portid, msg->msg_flags&MSG_DONTWAIT);
L
Linus Torvalds 已提交
2124 2125

out:
2126
	scm_destroy(siocb->scm);
L
Linus Torvalds 已提交
2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
	return err;
}

static int netlink_recvmsg(struct kiocb *kiocb, struct socket *sock,
			   struct msghdr *msg, size_t len,
			   int flags)
{
	struct sock_iocb *siocb = kiocb_to_siocb(kiocb);
	struct scm_cookie scm;
	struct sock *sk = sock->sk;
	struct netlink_sock *nlk = nlk_sk(sk);
	int noblock = flags&MSG_DONTWAIT;
	size_t copied;
J
Johannes Berg 已提交
2140
	struct sk_buff *skb, *data_skb;
2141
	int err, ret;
L
Linus Torvalds 已提交
2142 2143 2144 2145 2146 2147

	if (flags&MSG_OOB)
		return -EOPNOTSUPP;

	copied = 0;

2148 2149
	skb = skb_recv_datagram(sk, flags, noblock, &err);
	if (skb == NULL)
L
Linus Torvalds 已提交
2150 2151
		goto out;

J
Johannes Berg 已提交
2152 2153
	data_skb = skb;

2154 2155 2156
#ifdef CONFIG_COMPAT_NETLINK_MESSAGES
	if (unlikely(skb_shinfo(skb)->frag_list)) {
		/*
J
Johannes Berg 已提交
2157 2158 2159
		 * If this skb has a frag_list, then here that means that we
		 * will have to use the frag_list skb's data for compat tasks
		 * and the regular skb's data for normal (non-compat) tasks.
2160
		 *
J
Johannes Berg 已提交
2161 2162 2163 2164
		 * If we need to send the compat skb, assign it to the
		 * 'data_skb' variable so that it will be used below for data
		 * copying. We keep 'skb' for everything else, including
		 * freeing both later.
2165
		 */
J
Johannes Berg 已提交
2166 2167
		if (flags & MSG_CMSG_COMPAT)
			data_skb = skb_shinfo(skb)->frag_list;
2168 2169 2170
	}
#endif

L
Linus Torvalds 已提交
2171 2172
	msg->msg_namelen = 0;

J
Johannes Berg 已提交
2173
	copied = data_skb->len;
L
Linus Torvalds 已提交
2174 2175 2176 2177 2178
	if (len < copied) {
		msg->msg_flags |= MSG_TRUNC;
		copied = len;
	}

J
Johannes Berg 已提交
2179 2180
	skb_reset_transport_header(data_skb);
	err = skb_copy_datagram_iovec(data_skb, 0, msg->msg_iov, copied);
L
Linus Torvalds 已提交
2181 2182

	if (msg->msg_name) {
2183
		struct sockaddr_nl *addr = (struct sockaddr_nl *)msg->msg_name;
L
Linus Torvalds 已提交
2184 2185
		addr->nl_family = AF_NETLINK;
		addr->nl_pad    = 0;
2186
		addr->nl_pid	= NETLINK_CB(skb).portid;
2187
		addr->nl_groups	= netlink_group_mask(NETLINK_CB(skb).dst_group);
L
Linus Torvalds 已提交
2188 2189 2190
		msg->msg_namelen = sizeof(*addr);
	}

2191 2192 2193
	if (nlk->flags & NETLINK_RECV_PKTINFO)
		netlink_cmsg_recv_pktinfo(msg, skb);

L
Linus Torvalds 已提交
2194 2195 2196 2197 2198
	if (NULL == siocb->scm) {
		memset(&scm, 0, sizeof(scm));
		siocb->scm = &scm;
	}
	siocb->scm->creds = *NETLINK_CREDS(skb);
2199
	if (flags & MSG_TRUNC)
J
Johannes Berg 已提交
2200
		copied = data_skb->len;
2201

L
Linus Torvalds 已提交
2202 2203
	skb_free_datagram(sk, skb);

2204 2205 2206 2207 2208 2209 2210
	if (nlk->cb && atomic_read(&sk->sk_rmem_alloc) <= sk->sk_rcvbuf / 2) {
		ret = netlink_dump(sk);
		if (ret) {
			sk->sk_err = ret;
			sk->sk_error_report(sk);
		}
	}
L
Linus Torvalds 已提交
2211 2212 2213 2214 2215 2216 2217 2218 2219

	scm_recv(sock, msg, siocb->scm, flags);
out:
	netlink_rcv_wake(sk);
	return err ? : copied;
}

static void netlink_data_ready(struct sock *sk, int len)
{
2220
	BUG();
L
Linus Torvalds 已提交
2221 2222 2223
}

/*
2224
 *	We export these functions to other modules. They provide a
L
Linus Torvalds 已提交
2225 2226 2227 2228 2229
 *	complete set of kernel non-blocking support for message
 *	queueing.
 */

struct sock *
2230 2231
__netlink_kernel_create(struct net *net, int unit, struct module *module,
			struct netlink_kernel_cfg *cfg)
L
Linus Torvalds 已提交
2232 2233 2234
{
	struct socket *sock;
	struct sock *sk;
2235
	struct netlink_sock *nlk;
2236
	struct listeners *listeners = NULL;
2237 2238
	struct mutex *cb_mutex = cfg ? cfg->cb_mutex : NULL;
	unsigned int groups;
L
Linus Torvalds 已提交
2239

2240
	BUG_ON(!nl_table);
L
Linus Torvalds 已提交
2241

2242
	if (unit < 0 || unit >= MAX_LINKS)
L
Linus Torvalds 已提交
2243 2244 2245 2246 2247
		return NULL;

	if (sock_create_lite(PF_NETLINK, SOCK_DGRAM, unit, &sock))
		return NULL;

2248 2249 2250 2251 2252 2253 2254 2255 2256 2257
	/*
	 * We have to just have a reference on the net from sk, but don't
	 * get_net it. Besides, we cannot get and then put the net here.
	 * So we create one inside init_net and the move it to net.
	 */

	if (__netlink_create(&init_net, sock, cb_mutex, unit) < 0)
		goto out_sock_release_nosk;

	sk = sock->sk;
2258
	sk_change_net(sk, net);
2259

2260
	if (!cfg || cfg->groups < 32)
2261
		groups = 32;
2262 2263
	else
		groups = cfg->groups;
2264

2265
	listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
2266 2267 2268
	if (!listeners)
		goto out_sock_release;

L
Linus Torvalds 已提交
2269
	sk->sk_data_ready = netlink_data_ready;
2270 2271
	if (cfg && cfg->input)
		nlk_sk(sk)->netlink_rcv = cfg->input;
L
Linus Torvalds 已提交
2272

2273
	if (netlink_insert(sk, net, 0))
2274
		goto out_sock_release;
2275

2276 2277
	nlk = nlk_sk(sk);
	nlk->flags |= NETLINK_KERNEL_SOCKET;
2278 2279

	netlink_table_grab();
2280 2281
	if (!nl_table[unit].registered) {
		nl_table[unit].groups = groups;
2282
		rcu_assign_pointer(nl_table[unit].listeners, listeners);
2283 2284
		nl_table[unit].cb_mutex = cb_mutex;
		nl_table[unit].module = module;
2285 2286 2287 2288
		if (cfg) {
			nl_table[unit].bind = cfg->bind;
			nl_table[unit].flags = cfg->flags;
		}
2289
		nl_table[unit].registered = 1;
2290 2291
	} else {
		kfree(listeners);
2292
		nl_table[unit].registered++;
2293
	}
2294
	netlink_table_ungrab();
2295 2296
	return sk;

2297
out_sock_release:
2298
	kfree(listeners);
2299
	netlink_kernel_release(sk);
2300 2301 2302
	return NULL;

out_sock_release_nosk:
2303
	sock_release(sock);
2304
	return NULL;
L
Linus Torvalds 已提交
2305
}
2306
EXPORT_SYMBOL(__netlink_kernel_create);
2307 2308 2309 2310

void
netlink_kernel_release(struct sock *sk)
{
2311
	sk_release_kernel(sk);
2312 2313 2314
}
EXPORT_SYMBOL(netlink_kernel_release);

2315
int __netlink_change_ngroups(struct sock *sk, unsigned int groups)
2316
{
2317
	struct listeners *new, *old;
2318 2319 2320 2321 2322 2323
	struct netlink_table *tbl = &nl_table[sk->sk_protocol];

	if (groups < 32)
		groups = 32;

	if (NLGRPSZ(tbl->groups) < NLGRPSZ(groups)) {
2324 2325
		new = kzalloc(sizeof(*new) + NLGRPSZ(groups), GFP_ATOMIC);
		if (!new)
2326
			return -ENOMEM;
2327
		old = nl_deref_protected(tbl->listeners);
2328 2329 2330
		memcpy(new->masks, old->masks, NLGRPSZ(tbl->groups));
		rcu_assign_pointer(tbl->listeners, new);

2331
		kfree_rcu(old, rcu);
2332 2333 2334
	}
	tbl->groups = groups;

2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
	return 0;
}

/**
 * netlink_change_ngroups - change number of multicast groups
 *
 * This changes the number of multicast groups that are available
 * on a certain netlink family. Note that it is not possible to
 * change the number of groups to below 32. Also note that it does
 * not implicitly call netlink_clear_multicast_users() when the
 * number of groups is reduced.
 *
 * @sk: The kernel netlink socket, as returned by netlink_kernel_create().
 * @groups: The new number of groups.
 */
int netlink_change_ngroups(struct sock *sk, unsigned int groups)
{
	int err;

	netlink_table_grab();
	err = __netlink_change_ngroups(sk, groups);
2356
	netlink_table_ungrab();
2357

2358 2359 2360
	return err;
}

2361 2362 2363 2364 2365
void __netlink_clear_multicast_users(struct sock *ksk, unsigned int group)
{
	struct sock *sk;
	struct netlink_table *tbl = &nl_table[ksk->sk_protocol];

2366
	sk_for_each_bound(sk, &tbl->mc_list)
2367 2368 2369
		netlink_update_socket_mc(nlk_sk(sk), group, 0);
}

2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380
/**
 * netlink_clear_multicast_users - kick off multicast listeners
 *
 * This function removes all listeners from the given group.
 * @ksk: The kernel netlink socket, as returned by
 *	netlink_kernel_create().
 * @group: The multicast group to clear.
 */
void netlink_clear_multicast_users(struct sock *ksk, unsigned int group)
{
	netlink_table_grab();
2381
	__netlink_clear_multicast_users(ksk, group);
2382 2383 2384
	netlink_table_ungrab();
}

2385
struct nlmsghdr *
2386
__nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq, int type, int len, int flags)
2387 2388
{
	struct nlmsghdr *nlh;
2389
	int size = nlmsg_msg_size(len);
2390 2391 2392 2393 2394

	nlh = (struct nlmsghdr*)skb_put(skb, NLMSG_ALIGN(size));
	nlh->nlmsg_type = type;
	nlh->nlmsg_len = size;
	nlh->nlmsg_flags = flags;
2395
	nlh->nlmsg_pid = portid;
2396 2397
	nlh->nlmsg_seq = seq;
	if (!__builtin_constant_p(size) || NLMSG_ALIGN(size) - size != 0)
2398
		memset(nlmsg_data(nlh) + len, 0, NLMSG_ALIGN(size) - size);
2399 2400 2401 2402
	return nlh;
}
EXPORT_SYMBOL(__nlmsg_put);

L
Linus Torvalds 已提交
2403 2404 2405 2406 2407 2408 2409 2410 2411
/*
 * It looks a bit ugly.
 * It would be better to create kernel thread.
 */

static int netlink_dump(struct sock *sk)
{
	struct netlink_sock *nlk = nlk_sk(sk);
	struct netlink_callback *cb;
2412
	struct sk_buff *skb = NULL;
L
Linus Torvalds 已提交
2413
	struct nlmsghdr *nlh;
2414
	int len, err = -ENOBUFS;
2415
	int alloc_size;
L
Linus Torvalds 已提交
2416

2417
	mutex_lock(nlk->cb_mutex);
L
Linus Torvalds 已提交
2418 2419 2420

	cb = nlk->cb;
	if (cb == NULL) {
2421 2422
		err = -EINVAL;
		goto errout_skb;
L
Linus Torvalds 已提交
2423 2424
	}

2425 2426
	alloc_size = max_t(int, cb->min_dump_alloc, NLMSG_GOODSIZE);

2427 2428 2429 2430
	if (!netlink_rx_is_mmaped(sk) &&
	    atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf)
		goto errout_skb;
	skb = netlink_alloc_skb(sk, alloc_size, nlk->portid, GFP_KERNEL);
2431
	if (!skb)
2432
		goto errout_skb;
2433
	netlink_skb_set_owner_r(skb, sk);
2434

L
Linus Torvalds 已提交
2435 2436 2437
	len = cb->dump(skb, cb);

	if (len > 0) {
2438
		mutex_unlock(nlk->cb_mutex);
2439 2440 2441

		if (sk_filter(sk, skb))
			kfree_skb(skb);
2442 2443
		else
			__netlink_sendskb(sk, skb);
L
Linus Torvalds 已提交
2444 2445 2446
		return 0;
	}

2447 2448 2449 2450
	nlh = nlmsg_put_answer(skb, cb, NLMSG_DONE, sizeof(len), NLM_F_MULTI);
	if (!nlh)
		goto errout_skb;

2451 2452
	nl_dump_check_consistent(cb, nlh);

2453 2454
	memcpy(nlmsg_data(nlh), &len, sizeof(len));

2455 2456
	if (sk_filter(sk, skb))
		kfree_skb(skb);
2457 2458
	else
		__netlink_sendskb(sk, skb);
L
Linus Torvalds 已提交
2459

2460 2461
	if (cb->done)
		cb->done(cb);
L
Linus Torvalds 已提交
2462
	nlk->cb = NULL;
2463
	mutex_unlock(nlk->cb_mutex);
L
Linus Torvalds 已提交
2464

2465
	module_put(cb->module);
2466
	netlink_consume_callback(cb);
L
Linus Torvalds 已提交
2467
	return 0;
2468

2469
errout_skb:
2470
	mutex_unlock(nlk->cb_mutex);
2471 2472
	kfree_skb(skb);
	return err;
L
Linus Torvalds 已提交
2473 2474
}

2475 2476 2477
int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
			 const struct nlmsghdr *nlh,
			 struct netlink_dump_control *control)
L
Linus Torvalds 已提交
2478 2479 2480 2481
{
	struct netlink_callback *cb;
	struct sock *sk;
	struct netlink_sock *nlk;
2482
	int ret;
L
Linus Torvalds 已提交
2483

2484
	cb = kzalloc(sizeof(*cb), GFP_KERNEL);
L
Linus Torvalds 已提交
2485 2486 2487
	if (cb == NULL)
		return -ENOBUFS;

2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500
	/* Memory mapped dump requests need to be copied to avoid looping
	 * on the pending state in netlink_mmap_sendmsg() while the CB hold
	 * a reference to the skb.
	 */
	if (netlink_skb_is_mmaped(skb)) {
		skb = skb_copy(skb, GFP_KERNEL);
		if (skb == NULL) {
			kfree(cb);
			return -ENOBUFS;
		}
	} else
		atomic_inc(&skb->users);

2501 2502
	cb->dump = control->dump;
	cb->done = control->done;
L
Linus Torvalds 已提交
2503
	cb->nlh = nlh;
2504
	cb->data = control->data;
2505
	cb->module = control->module;
2506
	cb->min_dump_alloc = control->min_dump_alloc;
L
Linus Torvalds 已提交
2507 2508
	cb->skb = skb;

2509
	sk = netlink_lookup(sock_net(ssk), ssk->sk_protocol, NETLINK_CB(skb).portid);
L
Linus Torvalds 已提交
2510 2511 2512 2513 2514
	if (sk == NULL) {
		netlink_destroy_callback(cb);
		return -ECONNREFUSED;
	}
	nlk = nlk_sk(sk);
2515

2516
	mutex_lock(nlk->cb_mutex);
2517
	/* A dump is in progress... */
2518
	if (nlk->cb) {
2519
		mutex_unlock(nlk->cb_mutex);
L
Linus Torvalds 已提交
2520
		netlink_destroy_callback(cb);
2521 2522
		ret = -EBUSY;
		goto out;
L
Linus Torvalds 已提交
2523
	}
2524 2525 2526 2527 2528 2529 2530 2531
	/* add reference of module which cb->dump belongs to */
	if (!try_module_get(cb->module)) {
		mutex_unlock(nlk->cb_mutex);
		netlink_destroy_callback(cb);
		ret = -EPROTONOSUPPORT;
		goto out;
	}

L
Linus Torvalds 已提交
2532
	nlk->cb = cb;
2533
	mutex_unlock(nlk->cb_mutex);
L
Linus Torvalds 已提交
2534

2535
	ret = netlink_dump(sk);
2536
out:
L
Linus Torvalds 已提交
2537
	sock_put(sk);
2538

2539 2540 2541
	if (ret)
		return ret;

2542 2543 2544 2545
	/* We successfully started a dump, by returning -EINTR we
	 * signal not to send ACK even if it was requested.
	 */
	return -EINTR;
L
Linus Torvalds 已提交
2546
}
2547
EXPORT_SYMBOL(__netlink_dump_start);
L
Linus Torvalds 已提交
2548 2549 2550 2551 2552 2553

void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err)
{
	struct sk_buff *skb;
	struct nlmsghdr *rep;
	struct nlmsgerr *errmsg;
2554
	size_t payload = sizeof(*errmsg);
L
Linus Torvalds 已提交
2555

2556 2557 2558
	/* error messages get the original request appened */
	if (err)
		payload += nlmsg_len(nlh);
L
Linus Torvalds 已提交
2559

2560 2561
	skb = netlink_alloc_skb(in_skb->sk, nlmsg_total_size(payload),
				NETLINK_CB(in_skb).portid, GFP_KERNEL);
L
Linus Torvalds 已提交
2562 2563 2564
	if (!skb) {
		struct sock *sk;

2565
		sk = netlink_lookup(sock_net(in_skb->sk),
2566
				    in_skb->sk->sk_protocol,
2567
				    NETLINK_CB(in_skb).portid);
L
Linus Torvalds 已提交
2568 2569 2570 2571 2572 2573 2574 2575
		if (sk) {
			sk->sk_err = ENOBUFS;
			sk->sk_error_report(sk);
			sock_put(sk);
		}
		return;
	}

2576
	rep = __nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2577
			  NLMSG_ERROR, payload, 0);
2578
	errmsg = nlmsg_data(rep);
L
Linus Torvalds 已提交
2579
	errmsg->error = err;
2580
	memcpy(&errmsg->msg, nlh, err ? nlh->nlmsg_len : sizeof(*nlh));
2581
	netlink_unicast(in_skb->sk, skb, NETLINK_CB(in_skb).portid, MSG_DONTWAIT);
L
Linus Torvalds 已提交
2582
}
2583
EXPORT_SYMBOL(netlink_ack);
L
Linus Torvalds 已提交
2584

2585
int netlink_rcv_skb(struct sk_buff *skb, int (*cb)(struct sk_buff *,
2586
						     struct nlmsghdr *))
2587 2588 2589 2590 2591
{
	struct nlmsghdr *nlh;
	int err;

	while (skb->len >= nlmsg_total_size(0)) {
2592 2593
		int msglen;

2594
		nlh = nlmsg_hdr(skb);
2595
		err = 0;
2596

2597
		if (nlh->nlmsg_len < NLMSG_HDRLEN || skb->len < nlh->nlmsg_len)
2598 2599
			return 0;

2600 2601
		/* Only requests are handled by the kernel */
		if (!(nlh->nlmsg_flags & NLM_F_REQUEST))
2602
			goto ack;
2603 2604 2605

		/* Skip control messages */
		if (nlh->nlmsg_type < NLMSG_MIN_TYPE)
2606
			goto ack;
2607

2608
		err = cb(skb, nlh);
2609 2610 2611 2612
		if (err == -EINTR)
			goto skip;

ack:
2613
		if (nlh->nlmsg_flags & NLM_F_ACK || err)
2614 2615
			netlink_ack(skb, nlh, err);

2616
skip:
2617
		msglen = NLMSG_ALIGN(nlh->nlmsg_len);
2618 2619 2620
		if (msglen > skb->len)
			msglen = skb->len;
		skb_pull(skb, msglen);
2621 2622 2623 2624
	}

	return 0;
}
2625
EXPORT_SYMBOL(netlink_rcv_skb);
2626

2627 2628 2629 2630
/**
 * nlmsg_notify - send a notification netlink message
 * @sk: netlink socket to use
 * @skb: notification message
2631
 * @portid: destination netlink portid for reports or 0
2632 2633 2634 2635
 * @group: destination multicast group or 0
 * @report: 1 to report back, 0 to disable
 * @flags: allocation flags
 */
2636
int nlmsg_notify(struct sock *sk, struct sk_buff *skb, u32 portid,
2637 2638 2639 2640 2641
		 unsigned int group, int report, gfp_t flags)
{
	int err = 0;

	if (group) {
2642
		int exclude_portid = 0;
2643 2644 2645

		if (report) {
			atomic_inc(&skb->users);
2646
			exclude_portid = portid;
2647 2648
		}

2649 2650
		/* errors reported via destination sk->sk_err, but propagate
		 * delivery errors if NETLINK_BROADCAST_ERROR flag is set */
2651
		err = nlmsg_multicast(sk, skb, exclude_portid, group, flags);
2652 2653
	}

2654 2655 2656
	if (report) {
		int err2;

2657
		err2 = nlmsg_unicast(sk, skb, portid);
2658 2659 2660
		if (!err || err == -ESRCH)
			err = err2;
	}
2661 2662 2663

	return err;
}
2664
EXPORT_SYMBOL(nlmsg_notify);
2665

L
Linus Torvalds 已提交
2666 2667
#ifdef CONFIG_PROC_FS
struct nl_seq_iter {
2668
	struct seq_net_private p;
L
Linus Torvalds 已提交
2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
	int link;
	int hash_idx;
};

static struct sock *netlink_seq_socket_idx(struct seq_file *seq, loff_t pos)
{
	struct nl_seq_iter *iter = seq->private;
	int i, j;
	struct sock *s;
	loff_t off = 0;

2680
	for (i = 0; i < MAX_LINKS; i++) {
2681
		struct nl_portid_hash *hash = &nl_table[i].hash;
L
Linus Torvalds 已提交
2682 2683

		for (j = 0; j <= hash->mask; j++) {
2684
			sk_for_each(s, &hash->table[j]) {
2685
				if (sock_net(s) != seq_file_net(seq))
2686
					continue;
L
Linus Torvalds 已提交
2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
				if (off == pos) {
					iter->link = i;
					iter->hash_idx = j;
					return s;
				}
				++off;
			}
		}
	}
	return NULL;
}

static void *netlink_seq_start(struct seq_file *seq, loff_t *pos)
2700
	__acquires(nl_table_lock)
L
Linus Torvalds 已提交
2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715
{
	read_lock(&nl_table_lock);
	return *pos ? netlink_seq_socket_idx(seq, *pos - 1) : SEQ_START_TOKEN;
}

static void *netlink_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
	struct sock *s;
	struct nl_seq_iter *iter;
	int i, j;

	++*pos;

	if (v == SEQ_START_TOKEN)
		return netlink_seq_socket_idx(seq, 0);
2716

2717 2718 2719 2720
	iter = seq->private;
	s = v;
	do {
		s = sk_next(s);
2721
	} while (s && sock_net(s) != seq_file_net(seq));
L
Linus Torvalds 已提交
2722 2723 2724 2725 2726 2727 2728
	if (s)
		return s;

	i = iter->link;
	j = iter->hash_idx + 1;

	do {
2729
		struct nl_portid_hash *hash = &nl_table[i].hash;
L
Linus Torvalds 已提交
2730 2731 2732

		for (; j <= hash->mask; j++) {
			s = sk_head(&hash->table[j]);
2733
			while (s && sock_net(s) != seq_file_net(seq))
2734
				s = sk_next(s);
L
Linus Torvalds 已提交
2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748
			if (s) {
				iter->link = i;
				iter->hash_idx = j;
				return s;
			}
		}

		j = 0;
	} while (++i < MAX_LINKS);

	return NULL;
}

static void netlink_seq_stop(struct seq_file *seq, void *v)
2749
	__releases(nl_table_lock)
L
Linus Torvalds 已提交
2750 2751 2752 2753 2754 2755 2756
{
	read_unlock(&nl_table_lock);
}


static int netlink_seq_show(struct seq_file *seq, void *v)
{
E
Eric Dumazet 已提交
2757
	if (v == SEQ_START_TOKEN) {
L
Linus Torvalds 已提交
2758 2759
		seq_puts(seq,
			 "sk       Eth Pid    Groups   "
2760
			 "Rmem     Wmem     Dump     Locks     Drops     Inode\n");
E
Eric Dumazet 已提交
2761
	} else {
L
Linus Torvalds 已提交
2762 2763 2764
		struct sock *s = v;
		struct netlink_sock *nlk = nlk_sk(s);

2765
		seq_printf(seq, "%pK %-3d %-6u %08x %-8d %-8d %pK %-8d %-8d %-8lu\n",
L
Linus Torvalds 已提交
2766 2767
			   s,
			   s->sk_protocol,
2768
			   nlk->portid,
2769
			   nlk->groups ? (u32)nlk->groups[0] : 0,
2770 2771
			   sk_rmem_alloc_get(s),
			   sk_wmem_alloc_get(s),
L
Linus Torvalds 已提交
2772
			   nlk->cb,
2773
			   atomic_read(&s->sk_refcnt),
2774 2775
			   atomic_read(&s->sk_drops),
			   sock_i_ino(s)
L
Linus Torvalds 已提交
2776 2777 2778 2779 2780 2781
			);

	}
	return 0;
}

2782
static const struct seq_operations netlink_seq_ops = {
L
Linus Torvalds 已提交
2783 2784 2785 2786 2787 2788 2789 2790 2791
	.start  = netlink_seq_start,
	.next   = netlink_seq_next,
	.stop   = netlink_seq_stop,
	.show   = netlink_seq_show,
};


static int netlink_seq_open(struct inode *inode, struct file *file)
{
2792 2793
	return seq_open_net(inode, file, &netlink_seq_ops,
				sizeof(struct nl_seq_iter));
2794 2795
}

2796
static const struct file_operations netlink_seq_fops = {
L
Linus Torvalds 已提交
2797 2798 2799 2800
	.owner		= THIS_MODULE,
	.open		= netlink_seq_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
2801
	.release	= seq_release_net,
L
Linus Torvalds 已提交
2802 2803 2804 2805 2806 2807
};

#endif

int netlink_register_notifier(struct notifier_block *nb)
{
2808
	return atomic_notifier_chain_register(&netlink_chain, nb);
L
Linus Torvalds 已提交
2809
}
2810
EXPORT_SYMBOL(netlink_register_notifier);
L
Linus Torvalds 已提交
2811 2812 2813

int netlink_unregister_notifier(struct notifier_block *nb)
{
2814
	return atomic_notifier_chain_unregister(&netlink_chain, nb);
L
Linus Torvalds 已提交
2815
}
2816
EXPORT_SYMBOL(netlink_unregister_notifier);
2817

2818
static const struct proto_ops netlink_ops = {
L
Linus Torvalds 已提交
2819 2820 2821 2822 2823 2824 2825 2826
	.family =	PF_NETLINK,
	.owner =	THIS_MODULE,
	.release =	netlink_release,
	.bind =		netlink_bind,
	.connect =	netlink_connect,
	.socketpair =	sock_no_socketpair,
	.accept =	sock_no_accept,
	.getname =	netlink_getname,
2827
	.poll =		netlink_poll,
L
Linus Torvalds 已提交
2828 2829 2830
	.ioctl =	sock_no_ioctl,
	.listen =	sock_no_listen,
	.shutdown =	sock_no_shutdown,
2831 2832
	.setsockopt =	netlink_setsockopt,
	.getsockopt =	netlink_getsockopt,
L
Linus Torvalds 已提交
2833 2834
	.sendmsg =	netlink_sendmsg,
	.recvmsg =	netlink_recvmsg,
2835
	.mmap =		netlink_mmap,
L
Linus Torvalds 已提交
2836 2837 2838
	.sendpage =	sock_no_sendpage,
};

2839
static const struct net_proto_family netlink_family_ops = {
L
Linus Torvalds 已提交
2840 2841 2842 2843 2844
	.family = PF_NETLINK,
	.create = netlink_create,
	.owner	= THIS_MODULE,	/* for consistency 8) */
};

2845
static int __net_init netlink_net_init(struct net *net)
2846 2847
{
#ifdef CONFIG_PROC_FS
2848
	if (!proc_create("netlink", 0, net->proc_net, &netlink_seq_fops))
2849 2850 2851 2852 2853
		return -ENOMEM;
#endif
	return 0;
}

2854
static void __net_exit netlink_net_exit(struct net *net)
2855 2856
{
#ifdef CONFIG_PROC_FS
2857
	remove_proc_entry("netlink", net->proc_net);
2858 2859 2860
#endif
}

2861 2862
static void __init netlink_add_usersock_entry(void)
{
2863
	struct listeners *listeners;
2864 2865
	int groups = 32;

2866
	listeners = kzalloc(sizeof(*listeners) + NLGRPSZ(groups), GFP_KERNEL);
2867
	if (!listeners)
2868
		panic("netlink_add_usersock_entry: Cannot allocate listeners\n");
2869 2870 2871 2872

	netlink_table_grab();

	nl_table[NETLINK_USERSOCK].groups = groups;
2873
	rcu_assign_pointer(nl_table[NETLINK_USERSOCK].listeners, listeners);
2874 2875
	nl_table[NETLINK_USERSOCK].module = THIS_MODULE;
	nl_table[NETLINK_USERSOCK].registered = 1;
2876
	nl_table[NETLINK_USERSOCK].flags = NL_CFG_F_NONROOT_SEND;
2877 2878 2879 2880

	netlink_table_ungrab();
}

2881
static struct pernet_operations __net_initdata netlink_net_ops = {
2882 2883 2884 2885
	.init = netlink_net_init,
	.exit = netlink_net_exit,
};

L
Linus Torvalds 已提交
2886 2887 2888
static int __init netlink_proto_init(void)
{
	int i;
2889
	unsigned long limit;
L
Linus Torvalds 已提交
2890 2891 2892 2893 2894 2895
	unsigned int order;
	int err = proto_register(&netlink_proto, 0);

	if (err != 0)
		goto out;

2896
	BUILD_BUG_ON(sizeof(struct netlink_skb_parms) > FIELD_SIZEOF(struct sk_buff, cb));
L
Linus Torvalds 已提交
2897

2898
	nl_table = kcalloc(MAX_LINKS, sizeof(*nl_table), GFP_KERNEL);
2899 2900
	if (!nl_table)
		goto panic;
L
Linus Torvalds 已提交
2901

2902 2903
	if (totalram_pages >= (128 * 1024))
		limit = totalram_pages >> (21 - PAGE_SHIFT);
L
Linus Torvalds 已提交
2904
	else
2905
		limit = totalram_pages >> (23 - PAGE_SHIFT);
L
Linus Torvalds 已提交
2906

2907 2908 2909
	order = get_bitmask_order(limit) - 1 + PAGE_SHIFT;
	limit = (1UL << order) / sizeof(struct hlist_head);
	order = get_bitmask_order(min(limit, (unsigned long)UINT_MAX)) - 1;
L
Linus Torvalds 已提交
2910 2911

	for (i = 0; i < MAX_LINKS; i++) {
2912
		struct nl_portid_hash *hash = &nl_table[i].hash;
L
Linus Torvalds 已提交
2913

2914
		hash->table = nl_portid_hash_zalloc(1 * sizeof(*hash->table));
L
Linus Torvalds 已提交
2915 2916
		if (!hash->table) {
			while (i-- > 0)
2917
				nl_portid_hash_free(nl_table[i].hash.table,
L
Linus Torvalds 已提交
2918 2919
						 1 * sizeof(*hash->table));
			kfree(nl_table);
2920
			goto panic;
L
Linus Torvalds 已提交
2921 2922 2923 2924 2925 2926 2927
		}
		hash->max_shift = order;
		hash->shift = 0;
		hash->mask = 0;
		hash->rehash_time = jiffies;
	}

2928 2929
	netlink_add_usersock_entry();

L
Linus Torvalds 已提交
2930
	sock_register(&netlink_family_ops);
2931
	register_pernet_subsys(&netlink_net_ops);
2932
	/* The netlink device handler may be needed early. */
L
Linus Torvalds 已提交
2933 2934 2935
	rtnetlink_init();
out:
	return err;
2936 2937
panic:
	panic("netlink_init: Cannot allocate nl_table\n");
L
Linus Torvalds 已提交
2938 2939 2940
}

core_initcall(netlink_proto_init);