module.c 111.3 KB
Newer Older
1
/*
L
Linus Torvalds 已提交
2
   Copyright (C) 2002 Richard Henderson
3
   Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
19
#include <linux/export.h>
20
#include <linux/extable.h>
L
Linus Torvalds 已提交
21
#include <linux/moduleloader.h>
22
#include <linux/trace_events.h>
L
Linus Torvalds 已提交
23
#include <linux/init.h>
24
#include <linux/kallsyms.h>
25
#include <linux/file.h>
26
#include <linux/fs.h>
R
Roland McGrath 已提交
27
#include <linux/sysfs.h>
28
#include <linux/kernel.h>
L
Linus Torvalds 已提交
29 30 31
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/elf.h>
32
#include <linux/proc_fs.h>
33
#include <linux/security.h>
L
Linus Torvalds 已提交
34 35 36 37
#include <linux/seq_file.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/rcupdate.h>
38
#include <linux/capability.h>
L
Linus Torvalds 已提交
39 40 41 42 43 44
#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/vermagic.h>
#include <linux/notifier.h>
A
Al Viro 已提交
45
#include <linux/sched.h>
L
Linus Torvalds 已提交
46
#include <linux/device.h>
47
#include <linux/string.h>
A
Arjan van de Ven 已提交
48
#include <linux/mutex.h>
49
#include <linux/rculist.h>
50
#include <linux/uaccess.h>
L
Linus Torvalds 已提交
51
#include <asm/cacheflush.h>
52
#include <linux/set_memory.h>
53
#include <asm/mmu_context.h>
54
#include <linux/license.h>
55
#include <asm/sections.h>
M
Mathieu Desnoyers 已提交
56
#include <linux/tracepoint.h>
57
#include <linux/ftrace.h>
58
#include <linux/livepatch.h>
59
#include <linux/async.h>
60
#include <linux/percpu.h>
C
Catalin Marinas 已提交
61
#include <linux/kmemleak.h>
62
#include <linux/jump_label.h>
63
#include <linux/pfn.h>
64
#include <linux/bsearch.h>
65
#include <linux/dynamic_debug.h>
66
#include <linux/audit.h>
67
#include <uapi/linux/module.h>
R
Rusty Russell 已提交
68
#include "module-internal.h"
L
Linus Torvalds 已提交
69

70 71 72
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>

L
Linus Torvalds 已提交
73 74 75 76
#ifndef ARCH_SHF_SMALL
#define ARCH_SHF_SMALL 0
#endif

77 78 79
/*
 * Modules' sections will be aligned on page boundaries
 * to ensure complete separation of code and data, but
80
 * only when CONFIG_STRICT_MODULE_RWX=y
81
 */
82
#ifdef CONFIG_STRICT_MODULE_RWX
83 84 85 86 87
# define debug_align(X) ALIGN(X, PAGE_SIZE)
#else
# define debug_align(X) (X)
#endif

L
Linus Torvalds 已提交
88 89 90
/* If this is set, the section belongs in the init part of the module */
#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))

91 92 93 94 95
/*
 * Mutex protects:
 * 1) List of modules (also safely readable with preempt_disable),
 * 2) module_use links,
 * 3) module_addr_min/module_addr_max.
96
 * (delete and add uses RCU list operations). */
97 98
DEFINE_MUTEX(module_mutex);
EXPORT_SYMBOL_GPL(module_mutex);
L
Linus Torvalds 已提交
99
static LIST_HEAD(modules);
100

101
#ifdef CONFIG_MODULES_TREE_LOOKUP
R
Rusty Russell 已提交
102

103 104 105 106
/*
 * Use a latched RB-tree for __module_address(); this allows us to use
 * RCU-sched lookups of the address from any context.
 *
107 108 109
 * This is conditional on PERF_EVENTS || TRACING because those can really hit
 * __module_address() hard by doing a lot of stack unwinding; potentially from
 * NMI context.
110 111 112
 */

static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
R
Rusty Russell 已提交
113
{
114
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
R
Rusty Russell 已提交
115

116
	return (unsigned long)layout->base;
117 118 119 120
}

static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
{
121
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
122

123
	return (unsigned long)layout->size;
124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
}

static __always_inline bool
mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
{
	return __mod_tree_val(a) < __mod_tree_val(b);
}

static __always_inline int
mod_tree_comp(void *key, struct latch_tree_node *n)
{
	unsigned long val = (unsigned long)key;
	unsigned long start, end;

	start = __mod_tree_val(n);
	if (val < start)
		return -1;

	end = start + __mod_tree_size(n);
	if (val >= end)
		return 1;
R
Rusty Russell 已提交
145 146 147 148

	return 0;
}

149 150 151 152 153
static const struct latch_tree_ops mod_tree_ops = {
	.less = mod_tree_less,
	.comp = mod_tree_comp,
};

154 155 156 157 158 159
static struct mod_tree_root {
	struct latch_tree_root root;
	unsigned long addr_min;
	unsigned long addr_max;
} mod_tree __cacheline_aligned = {
	.addr_min = -1UL,
R
Rusty Russell 已提交
160 161
};

162 163 164 165 166 167 168 169 170 171 172 173
#define module_addr_min mod_tree.addr_min
#define module_addr_max mod_tree.addr_max

static noinline void __mod_tree_insert(struct mod_tree_node *node)
{
	latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
}

static void __mod_tree_remove(struct mod_tree_node *node)
{
	latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
}
174 175 176 177 178 179 180

/*
 * These modifications: insert, remove_init and remove; are serialized by the
 * module_mutex.
 */
static void mod_tree_insert(struct module *mod)
{
181 182
	mod->core_layout.mtn.mod = mod;
	mod->init_layout.mtn.mod = mod;
183

184 185 186
	__mod_tree_insert(&mod->core_layout.mtn);
	if (mod->init_layout.size)
		__mod_tree_insert(&mod->init_layout.mtn);
187 188 189 190
}

static void mod_tree_remove_init(struct module *mod)
{
191 192
	if (mod->init_layout.size)
		__mod_tree_remove(&mod->init_layout.mtn);
193 194 195 196
}

static void mod_tree_remove(struct module *mod)
{
197
	__mod_tree_remove(&mod->core_layout.mtn);
198 199 200
	mod_tree_remove_init(mod);
}

201
static struct module *mod_find(unsigned long addr)
202 203 204
{
	struct latch_tree_node *ltn;

205
	ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
206 207 208 209 210 211
	if (!ltn)
		return NULL;

	return container_of(ltn, struct mod_tree_node, node)->mod;
}

212 213
#else /* MODULES_TREE_LOOKUP */

214 215
static unsigned long module_addr_min = -1UL, module_addr_max = 0;

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233
static void mod_tree_insert(struct module *mod) { }
static void mod_tree_remove_init(struct module *mod) { }
static void mod_tree_remove(struct module *mod) { }

static struct module *mod_find(unsigned long addr)
{
	struct module *mod;

	list_for_each_entry_rcu(mod, &modules, list) {
		if (within_module(addr, mod))
			return mod;
	}

	return NULL;
}

#endif /* MODULES_TREE_LOOKUP */

234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250
/*
 * Bounds of module text, for speeding up __module_address.
 * Protected by module_mutex.
 */
static void __mod_update_bounds(void *base, unsigned int size)
{
	unsigned long min = (unsigned long)base;
	unsigned long max = min + size;

	if (min < module_addr_min)
		module_addr_min = min;
	if (max > module_addr_max)
		module_addr_max = max;
}

static void mod_update_bounds(struct module *mod)
{
251 252 253
	__mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
	if (mod->init_layout.size)
		__mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
254 255
}

256 257 258 259
#ifdef CONFIG_KGDB_KDB
struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
#endif /* CONFIG_KGDB_KDB */

260 261 262 263 264 265 266 267 268 269 270
static void module_assert_mutex(void)
{
	lockdep_assert_held(&module_mutex);
}

static void module_assert_mutex_or_preempt(void)
{
#ifdef CONFIG_LOCKDEP
	if (unlikely(!debug_locks))
		return;

271
	WARN_ON_ONCE(!rcu_read_lock_sched_held() &&
272 273 274 275
		!lockdep_is_held(&module_mutex));
#endif
}

276 277
static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
#ifndef CONFIG_MODULE_SIG_FORCE
R
Rusty Russell 已提交
278 279
module_param(sig_enforce, bool_enable_only, 0644);
#endif /* !CONFIG_MODULE_SIG_FORCE */
L
Linus Torvalds 已提交
280

281 282 283 284 285 286 287 288 289 290
/*
 * Export sig_enforce kernel cmdline parameter to allow other subsystems rely
 * on that instead of directly to CONFIG_MODULE_SIG_FORCE config.
 */
bool is_module_sig_enforced(void)
{
	return sig_enforce;
}
EXPORT_SYMBOL(is_module_sig_enforced);

291 292
/* Block module loading/unloading? */
int modules_disabled = 0;
293
core_param(nomodule, modules_disabled, bint, 0);
294

295 296 297
/* Waiting for a module to finish initializing? */
static DECLARE_WAIT_QUEUE_HEAD(module_wq);

298
static BLOCKING_NOTIFIER_HEAD(module_notify_list);
L
Linus Torvalds 已提交
299

300
int register_module_notifier(struct notifier_block *nb)
L
Linus Torvalds 已提交
301
{
302
	return blocking_notifier_chain_register(&module_notify_list, nb);
L
Linus Torvalds 已提交
303 304 305
}
EXPORT_SYMBOL(register_module_notifier);

306
int unregister_module_notifier(struct notifier_block *nb)
L
Linus Torvalds 已提交
307
{
308
	return blocking_notifier_chain_unregister(&module_notify_list, nb);
L
Linus Torvalds 已提交
309 310 311
}
EXPORT_SYMBOL(unregister_module_notifier);

312
struct load_info {
313
	const char *name;
314 315 316
	Elf_Ehdr *hdr;
	unsigned long len;
	Elf_Shdr *sechdrs;
317
	char *secstrings, *strtab;
R
Rusty Russell 已提交
318
	unsigned long symoffs, stroffs;
319 320
	struct _ddebug *debug;
	unsigned int num_debug;
R
Rusty Russell 已提交
321
	bool sig_ok;
322 323 324
#ifdef CONFIG_KALLSYMS
	unsigned long mod_kallsyms_init_off;
#endif
325 326 327 328 329
	struct {
		unsigned int sym, str, mod, vers, info, pcpu;
	} index;
};

330 331 332 333 334
/*
 * We require a truly strong try_module_get(): 0 means success.
 * Otherwise an error is returned due to ongoing or failed
 * initialization etc.
 */
L
Linus Torvalds 已提交
335 336
static inline int strong_try_module_get(struct module *mod)
{
337
	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
L
Linus Torvalds 已提交
338
	if (mod && mod->state == MODULE_STATE_COMING)
339 340
		return -EBUSY;
	if (try_module_get(mod))
L
Linus Torvalds 已提交
341
		return 0;
342 343
	else
		return -ENOENT;
L
Linus Torvalds 已提交
344 345
}

346 347
static inline void add_taint_module(struct module *mod, unsigned flag,
				    enum lockdep_ok lockdep_ok)
348
{
349
	add_taint(flag, lockdep_ok);
350
	set_bit(flag, &mod->taints);
351 352
}

353 354 355
/*
 * A thread that wants to hold a reference to a module only while it
 * is running can call this to safely exit.  nfsd and lockd use this.
L
Linus Torvalds 已提交
356
 */
357
void __noreturn __module_put_and_exit(struct module *mod, long code)
L
Linus Torvalds 已提交
358 359 360 361 362
{
	module_put(mod);
	do_exit(code);
}
EXPORT_SYMBOL(__module_put_and_exit);
D
Daniel Walker 已提交
363

L
Linus Torvalds 已提交
364
/* Find a module section: 0 means not found. */
365
static unsigned int find_sec(const struct load_info *info, const char *name)
L
Linus Torvalds 已提交
366 367 368
{
	unsigned int i;

369 370
	for (i = 1; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *shdr = &info->sechdrs[i];
L
Linus Torvalds 已提交
371
		/* Alloc bit cleared means "ignore it." */
372 373
		if ((shdr->sh_flags & SHF_ALLOC)
		    && strcmp(info->secstrings + shdr->sh_name, name) == 0)
L
Linus Torvalds 已提交
374
			return i;
375
	}
L
Linus Torvalds 已提交
376 377 378
	return 0;
}

R
Rusty Russell 已提交
379
/* Find a module section, or NULL. */
380
static void *section_addr(const struct load_info *info, const char *name)
R
Rusty Russell 已提交
381 382
{
	/* Section 0 has sh_addr 0. */
383
	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
R
Rusty Russell 已提交
384 385 386
}

/* Find a module section, or NULL.  Fill in number of "objects" in section. */
387
static void *section_objs(const struct load_info *info,
R
Rusty Russell 已提交
388 389 390 391
			  const char *name,
			  size_t object_size,
			  unsigned int *num)
{
392
	unsigned int sec = find_sec(info, name);
R
Rusty Russell 已提交
393 394

	/* Section 0 has sh_addr 0 and sh_size 0. */
395 396
	*num = info->sechdrs[sec].sh_size / object_size;
	return (void *)info->sechdrs[sec].sh_addr;
R
Rusty Russell 已提交
397 398
}

L
Linus Torvalds 已提交
399 400 401 402 403
/* Provided by the linker */
extern const struct kernel_symbol __start___ksymtab[];
extern const struct kernel_symbol __stop___ksymtab[];
extern const struct kernel_symbol __start___ksymtab_gpl[];
extern const struct kernel_symbol __stop___ksymtab_gpl[];
404 405
extern const struct kernel_symbol __start___ksymtab_gpl_future[];
extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
406 407 408
extern const s32 __start___kcrctab[];
extern const s32 __start___kcrctab_gpl[];
extern const s32 __start___kcrctab_gpl_future[];
409 410 411 412 413
#ifdef CONFIG_UNUSED_SYMBOLS
extern const struct kernel_symbol __start___ksymtab_unused[];
extern const struct kernel_symbol __stop___ksymtab_unused[];
extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
414 415
extern const s32 __start___kcrctab_unused[];
extern const s32 __start___kcrctab_unused_gpl[];
416
#endif
L
Linus Torvalds 已提交
417 418 419 420

#ifndef CONFIG_MODVERSIONS
#define symversion(base, idx) NULL
#else
A
Andrew Morton 已提交
421
#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
L
Linus Torvalds 已提交
422 423
#endif

424 425 426 427 428
static bool each_symbol_in_section(const struct symsearch *arr,
				   unsigned int arrsize,
				   struct module *owner,
				   bool (*fn)(const struct symsearch *syms,
					      struct module *owner,
429
					      void *data),
430
				   void *data)
431
{
432
	unsigned int j;
433

434
	for (j = 0; j < arrsize; j++) {
435 436
		if (fn(&arr[j], owner, data))
			return true;
437
	}
438 439

	return false;
440 441
}

442
/* Returns true as soon as fn returns true, otherwise false. */
443 444 445 446
bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
				    struct module *owner,
				    void *data),
			 void *data)
447 448
{
	struct module *mod;
449
	static const struct symsearch arr[] = {
450
		{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
451
		  NOT_GPL_ONLY, false },
452
		{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
453 454
		  __start___kcrctab_gpl,
		  GPL_ONLY, false },
455
		{ __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
456 457
		  __start___kcrctab_gpl_future,
		  WILL_BE_GPL_ONLY, false },
458
#ifdef CONFIG_UNUSED_SYMBOLS
459
		{ __start___ksymtab_unused, __stop___ksymtab_unused,
460 461
		  __start___kcrctab_unused,
		  NOT_GPL_ONLY, true },
462
		{ __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
463 464
		  __start___kcrctab_unused_gpl,
		  GPL_ONLY, true },
465
#endif
466
	};
467

468 469
	module_assert_mutex_or_preempt();

470 471
	if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
		return true;
472

473
	list_for_each_entry_rcu(mod, &modules, list) {
474 475
		struct symsearch arr[] = {
			{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
476
			  NOT_GPL_ONLY, false },
477
			{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
478 479
			  mod->gpl_crcs,
			  GPL_ONLY, false },
480 481
			{ mod->gpl_future_syms,
			  mod->gpl_future_syms + mod->num_gpl_future_syms,
482 483
			  mod->gpl_future_crcs,
			  WILL_BE_GPL_ONLY, false },
484
#ifdef CONFIG_UNUSED_SYMBOLS
485 486
			{ mod->unused_syms,
			  mod->unused_syms + mod->num_unused_syms,
487 488
			  mod->unused_crcs,
			  NOT_GPL_ONLY, true },
489 490
			{ mod->unused_gpl_syms,
			  mod->unused_gpl_syms + mod->num_unused_gpl_syms,
491 492
			  mod->unused_gpl_crcs,
			  GPL_ONLY, true },
493
#endif
494 495
		};

496 497 498
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;

499 500 501 502 503
		if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
			return true;
	}
	return false;
}
504
EXPORT_SYMBOL_GPL(each_symbol_section);
505 506 507 508 509 510 511 512 513

struct find_symbol_arg {
	/* Input */
	const char *name;
	bool gplok;
	bool warn;

	/* Output */
	struct module *owner;
514
	const s32 *crc;
515
	const struct kernel_symbol *sym;
516 517
};

518 519 520
static bool check_symbol(const struct symsearch *syms,
				 struct module *owner,
				 unsigned int symnum, void *data)
521 522 523 524 525 526 527
{
	struct find_symbol_arg *fsa = data;

	if (!fsa->gplok) {
		if (syms->licence == GPL_ONLY)
			return false;
		if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
A
Andrew Morton 已提交
528 529 530
			pr_warn("Symbol %s is being used by a non-GPL module, "
				"which will not be allowed in the future\n",
				fsa->name);
531
		}
L
Linus Torvalds 已提交
532
	}
533

534
#ifdef CONFIG_UNUSED_SYMBOLS
535
	if (syms->unused && fsa->warn) {
A
Andrew Morton 已提交
536 537 538
		pr_warn("Symbol %s is marked as UNUSED, however this module is "
			"using it.\n", fsa->name);
		pr_warn("This symbol will go away in the future.\n");
539 540 541
		pr_warn("Please evaluate if this is the right api to use and "
			"if it really is, submit a report to the linux kernel "
			"mailing list together with submitting your code for "
A
Andrew Morton 已提交
542
			"inclusion.\n");
543
	}
544
#endif
545 546 547

	fsa->owner = owner;
	fsa->crc = symversion(syms->crcs, symnum);
548
	fsa->sym = &syms->start[symnum];
549 550 551
	return true;
}

552 553 554 555 556 557 558 559
static int cmp_name(const void *va, const void *vb)
{
	const char *a;
	const struct kernel_symbol *b;
	a = va; b = vb;
	return strcmp(a, b->name);
}

560 561 562 563 564
static bool find_symbol_in_section(const struct symsearch *syms,
				   struct module *owner,
				   void *data)
{
	struct find_symbol_arg *fsa = data;
565 566 567 568 569 570 571
	struct kernel_symbol *sym;

	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
			sizeof(struct kernel_symbol), cmp_name);

	if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
		return true;
572 573 574 575

	return false;
}

576
/* Find a symbol and return it, along with, (optional) crc and
577
 * (optional) module which owns it.  Needs preempt disabled or module_mutex. */
578 579
const struct kernel_symbol *find_symbol(const char *name,
					struct module **owner,
580
					const s32 **crc,
581 582
					bool gplok,
					bool warn)
583 584 585 586 587 588 589
{
	struct find_symbol_arg fsa;

	fsa.name = name;
	fsa.gplok = gplok;
	fsa.warn = warn;

590
	if (each_symbol_section(find_symbol_in_section, &fsa)) {
591 592 593 594
		if (owner)
			*owner = fsa.owner;
		if (crc)
			*crc = fsa.crc;
595
		return fsa.sym;
596 597
	}

598
	pr_debug("Failed to find symbol %s\n", name);
599
	return NULL;
L
Linus Torvalds 已提交
600
}
601
EXPORT_SYMBOL_GPL(find_symbol);
L
Linus Torvalds 已提交
602

603 604 605 606
/*
 * Search for module by name: must hold module_mutex (or preempt disabled
 * for read-only access).
 */
607
static struct module *find_module_all(const char *name, size_t len,
608
				      bool even_unformed)
L
Linus Torvalds 已提交
609 610 611
{
	struct module *mod;

612
	module_assert_mutex_or_preempt();
613

614
	list_for_each_entry_rcu(mod, &modules, list) {
615 616
		if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
			continue;
617
		if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
L
Linus Torvalds 已提交
618 619 620 621
			return mod;
	}
	return NULL;
}
622 623 624

struct module *find_module(const char *name)
{
625
	module_assert_mutex();
626
	return find_module_all(name, strlen(name), false);
627
}
628
EXPORT_SYMBOL_GPL(find_module);
L
Linus Torvalds 已提交
629 630

#ifdef CONFIG_SMP
631

632
static inline void __percpu *mod_percpu(struct module *mod)
633
{
634 635
	return mod->percpu;
}
636

R
Rusty Russell 已提交
637
static int percpu_modalloc(struct module *mod, struct load_info *info)
638
{
R
Rusty Russell 已提交
639 640 641 642 643 644
	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
	unsigned long align = pcpusec->sh_addralign;

	if (!pcpusec->sh_size)
		return 0;

645
	if (align > PAGE_SIZE) {
A
Andrew Morton 已提交
646 647
		pr_warn("%s: per-cpu alignment %li > %li\n",
			mod->name, align, PAGE_SIZE);
648 649 650
		align = PAGE_SIZE;
	}

R
Rusty Russell 已提交
651
	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
652
	if (!mod->percpu) {
A
Andrew Morton 已提交
653 654
		pr_warn("%s: Could not allocate %lu bytes percpu data\n",
			mod->name, (unsigned long)pcpusec->sh_size);
655 656
		return -ENOMEM;
	}
R
Rusty Russell 已提交
657
	mod->percpu_size = pcpusec->sh_size;
658
	return 0;
659 660
}

661
static void percpu_modfree(struct module *mod)
662
{
663
	free_percpu(mod->percpu);
664 665
}

666
static unsigned int find_pcpusec(struct load_info *info)
667
{
668
	return find_sec(info, ".data..percpu");
669 670
}

671 672
static void percpu_modcopy(struct module *mod,
			   const void *from, unsigned long size)
673 674 675 676
{
	int cpu;

	for_each_possible_cpu(cpu)
677
		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
678 679
}

680
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
681 682 683 684 685 686 687
{
	struct module *mod;
	unsigned int cpu;

	preempt_disable();

	list_for_each_entry_rcu(mod, &modules, list) {
688 689
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
690 691 692 693
		if (!mod->percpu_size)
			continue;
		for_each_possible_cpu(cpu) {
			void *start = per_cpu_ptr(mod->percpu, cpu);
694
			void *va = (void *)addr;
695

696
			if (va >= start && va < start + mod->percpu_size) {
697
				if (can_addr) {
698
					*can_addr = (unsigned long) (va - start);
699 700 701 702
					*can_addr += (unsigned long)
						per_cpu_ptr(mod->percpu,
							    get_boot_cpu_id());
				}
703 704 705 706 707 708 709 710
				preempt_enable();
				return true;
			}
		}
	}

	preempt_enable();
	return false;
711 712
}

713 714 715 716 717 718 719 720 721 722 723 724 725 726
/**
 * is_module_percpu_address - test whether address is from module static percpu
 * @addr: address to test
 *
 * Test whether @addr belongs to module static percpu area.
 *
 * RETURNS:
 * %true if @addr is from module static percpu area
 */
bool is_module_percpu_address(unsigned long addr)
{
	return __is_module_percpu_address(addr, NULL);
}

L
Linus Torvalds 已提交
727
#else /* ... !CONFIG_SMP */
728

729
static inline void __percpu *mod_percpu(struct module *mod)
L
Linus Torvalds 已提交
730 731 732
{
	return NULL;
}
R
Rusty Russell 已提交
733
static int percpu_modalloc(struct module *mod, struct load_info *info)
734
{
R
Rusty Russell 已提交
735 736 737 738
	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
	if (info->sechdrs[info->index.pcpu].sh_size != 0)
		return -ENOMEM;
	return 0;
739 740
}
static inline void percpu_modfree(struct module *mod)
L
Linus Torvalds 已提交
741 742
{
}
743
static unsigned int find_pcpusec(struct load_info *info)
L
Linus Torvalds 已提交
744 745 746
{
	return 0;
}
747 748
static inline void percpu_modcopy(struct module *mod,
				  const void *from, unsigned long size)
L
Linus Torvalds 已提交
749 750 751 752
{
	/* pcpusec should be 0, and size of that section should be 0. */
	BUG_ON(size != 0);
}
753 754 755 756
bool is_module_percpu_address(unsigned long addr)
{
	return false;
}
757

758 759 760 761 762
bool __is_module_percpu_address(unsigned long addr, unsigned long *can_addr)
{
	return false;
}

L
Linus Torvalds 已提交
763 764
#endif /* CONFIG_SMP */

765 766 767 768 769 770
#define MODINFO_ATTR(field)	\
static void setup_modinfo_##field(struct module *mod, const char *s)  \
{                                                                     \
	mod->field = kstrdup(s, GFP_KERNEL);                          \
}                                                                     \
static ssize_t show_modinfo_##field(struct module_attribute *mattr,   \
771
			struct module_kobject *mk, char *buffer)      \
772
{                                                                     \
773
	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field);  \
774 775 776 777 778 779 780
}                                                                     \
static int modinfo_##field##_exists(struct module *mod)               \
{                                                                     \
	return mod->field != NULL;                                    \
}                                                                     \
static void free_modinfo_##field(struct module *mod)                  \
{                                                                     \
D
Daniel Walker 已提交
781 782
	kfree(mod->field);                                            \
	mod->field = NULL;                                            \
783 784
}                                                                     \
static struct module_attribute modinfo_##field = {                    \
785
	.attr = { .name = __stringify(field), .mode = 0444 },         \
786 787 788 789 790 791 792 793 794
	.show = show_modinfo_##field,                                 \
	.setup = setup_modinfo_##field,                               \
	.test = modinfo_##field##_exists,                             \
	.free = free_modinfo_##field,                                 \
};

MODINFO_ATTR(version);
MODINFO_ATTR(srcversion);

795 796
static char last_unloaded_module[MODULE_NAME_LEN+1];

797
#ifdef CONFIG_MODULE_UNLOAD
798 799 800

EXPORT_TRACEPOINT_SYMBOL(module_get);

801 802 803
/* MODULE_REF_BASE is the base reference count by kmodule loader. */
#define MODULE_REF_BASE	1

L
Linus Torvalds 已提交
804
/* Init the unload section of the module. */
805
static int module_unload_init(struct module *mod)
L
Linus Torvalds 已提交
806
{
807 808 809 810 811
	/*
	 * Initialize reference counter to MODULE_REF_BASE.
	 * refcnt == 0 means module is going.
	 */
	atomic_set(&mod->refcnt, MODULE_REF_BASE);
812

813 814
	INIT_LIST_HEAD(&mod->source_list);
	INIT_LIST_HEAD(&mod->target_list);
815

L
Linus Torvalds 已提交
816
	/* Hold reference count during initialization. */
817
	atomic_inc(&mod->refcnt);
818 819

	return 0;
L
Linus Torvalds 已提交
820 821 822 823 824 825 826
}

/* Does a already use b? */
static int already_uses(struct module *a, struct module *b)
{
	struct module_use *use;

827 828
	list_for_each_entry(use, &b->source_list, source_list) {
		if (use->source == a) {
829
			pr_debug("%s uses %s!\n", a->name, b->name);
L
Linus Torvalds 已提交
830 831 832
			return 1;
		}
	}
833
	pr_debug("%s does not use %s!\n", a->name, b->name);
L
Linus Torvalds 已提交
834 835 836
	return 0;
}

837 838 839 840 841 842 843 844 845 846 847
/*
 * Module a uses b
 *  - we add 'a' as a "source", 'b' as a "target" of module use
 *  - the module_use is added to the list of 'b' sources (so
 *    'b' can walk the list to see who sourced them), and of 'a'
 *    targets (so 'a' can see what modules it targets).
 */
static int add_module_usage(struct module *a, struct module *b)
{
	struct module_use *use;

848
	pr_debug("Allocating new usage for %s.\n", a->name);
849
	use = kmalloc(sizeof(*use), GFP_ATOMIC);
850
	if (!use)
851 852 853 854 855 856 857 858 859
		return -ENOMEM;

	use->source = a;
	use->target = b;
	list_add(&use->source_list, &b->source_list);
	list_add(&use->target_list, &a->target_list);
	return 0;
}

860
/* Module a uses b: caller needs module_mutex() */
861
int ref_module(struct module *a, struct module *b)
L
Linus Torvalds 已提交
862
{
863
	int err;
K
Kay Sievers 已提交
864

865
	if (b == NULL || already_uses(a, b))
866 867
		return 0;

868 869
	/* If module isn't available, we fail. */
	err = strong_try_module_get(b);
870
	if (err)
871
		return err;
L
Linus Torvalds 已提交
872

873 874
	err = add_module_usage(a, b);
	if (err) {
L
Linus Torvalds 已提交
875
		module_put(b);
876
		return err;
L
Linus Torvalds 已提交
877
	}
878
	return 0;
L
Linus Torvalds 已提交
879
}
880
EXPORT_SYMBOL_GPL(ref_module);
L
Linus Torvalds 已提交
881 882 883 884

/* Clear the unload stuff of the module. */
static void module_unload_free(struct module *mod)
{
885
	struct module_use *use, *tmp;
L
Linus Torvalds 已提交
886

887
	mutex_lock(&module_mutex);
888 889
	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
		struct module *i = use->target;
890
		pr_debug("%s unusing %s\n", mod->name, i->name);
891 892 893 894
		module_put(i);
		list_del(&use->source_list);
		list_del(&use->target_list);
		kfree(use);
L
Linus Torvalds 已提交
895
	}
896
	mutex_unlock(&module_mutex);
L
Linus Torvalds 已提交
897 898 899
}

#ifdef CONFIG_MODULE_FORCE_UNLOAD
900
static inline int try_force_unload(unsigned int flags)
L
Linus Torvalds 已提交
901 902 903
{
	int ret = (flags & O_TRUNC);
	if (ret)
904
		add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
905 906 907
	return ret;
}
#else
908
static inline int try_force_unload(unsigned int flags)
L
Linus Torvalds 已提交
909 910 911 912 913
{
	return 0;
}
#endif /* CONFIG_MODULE_FORCE_UNLOAD */

914 915
/* Try to release refcount of module, 0 means success. */
static int try_release_module_ref(struct module *mod)
L
Linus Torvalds 已提交
916
{
917
	int ret;
L
Linus Torvalds 已提交
918

919 920 921 922 923 924
	/* Try to decrement refcnt which we set at loading */
	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
	BUG_ON(ret < 0);
	if (ret)
		/* Someone can put this right now, recover with checking */
		ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
L
Linus Torvalds 已提交
925

926 927
	return ret;
}
L
Linus Torvalds 已提交
928

929 930
static int try_stop_module(struct module *mod, int flags, int *forced)
{
931
	/* If it's not unused, quit unless we're forcing. */
932 933 934
	if (try_release_module_ref(mod) != 0) {
		*forced = try_force_unload(flags);
		if (!(*forced))
L
Linus Torvalds 已提交
935 936 937 938
			return -EWOULDBLOCK;
	}

	/* Mark it as dying. */
939
	mod->state = MODULE_STATE_GOING;
L
Linus Torvalds 已提交
940

941
	return 0;
L
Linus Torvalds 已提交
942 943
}

944 945 946 947 948 949 950 951 952 953
/**
 * module_refcount - return the refcount or -1 if unloading
 *
 * @mod:	the module we're checking
 *
 * Returns:
 *	-1 if the module is in the process of unloading
 *	otherwise the number of references in the kernel to the module
 */
int module_refcount(struct module *mod)
L
Linus Torvalds 已提交
954
{
955
	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
L
Linus Torvalds 已提交
956 957 958 959 960 961
}
EXPORT_SYMBOL(module_refcount);

/* This exists whether we can unload or not */
static void free_module(struct module *mod);

962 963
SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
		unsigned int, flags)
L
Linus Torvalds 已提交
964 965
{
	struct module *mod;
966
	char name[MODULE_NAME_LEN];
L
Linus Torvalds 已提交
967 968
	int ret, forced = 0;

969
	if (!capable(CAP_SYS_MODULE) || modules_disabled)
970 971 972 973 974 975
		return -EPERM;

	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
		return -EFAULT;
	name[MODULE_NAME_LEN-1] = '\0';

976 977
	audit_log_kern_module(name);

978 979
	if (mutex_lock_interruptible(&module_mutex) != 0)
		return -EINTR;
L
Linus Torvalds 已提交
980 981 982 983 984 985 986

	mod = find_module(name);
	if (!mod) {
		ret = -ENOENT;
		goto out;
	}

987
	if (!list_empty(&mod->source_list)) {
L
Linus Torvalds 已提交
988 989 990 991 992 993 994
		/* Other modules depend on us: get rid of them first. */
		ret = -EWOULDBLOCK;
		goto out;
	}

	/* Doing init or already dying? */
	if (mod->state != MODULE_STATE_LIVE) {
995
		/* FIXME: if (force), slam module count damn the torpedoes */
996
		pr_debug("%s already dying\n", mod->name);
L
Linus Torvalds 已提交
997 998 999 1000 1001
		ret = -EBUSY;
		goto out;
	}

	/* If it has an init func, it must have an exit func to unload */
R
Rusty Russell 已提交
1002
	if (mod->init && !mod->exit) {
1003
		forced = try_force_unload(flags);
L
Linus Torvalds 已提交
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
		if (!forced) {
			/* This module can't be removed */
			ret = -EBUSY;
			goto out;
		}
	}

	/* Stop the machine so refcounts can't move and disable module. */
	ret = try_stop_module(mod, flags, &forced);
	if (ret != 0)
		goto out;

1016
	mutex_unlock(&module_mutex);
L
Lucas De Marchi 已提交
1017
	/* Final destruction now no one is using it. */
1018
	if (mod->exit != NULL)
L
Linus Torvalds 已提交
1019
		mod->exit();
1020 1021
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
1022
	klp_module_going(mod);
1023 1024
	ftrace_release_mod(mod);

1025
	async_synchronize_full();
1026

1027
	/* Store the name of the last unloaded module for diagnostic purposes */
1028
	strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
L
Linus Torvalds 已提交
1029

1030 1031 1032
	free_module(mod);
	return 0;
out:
1033
	mutex_unlock(&module_mutex);
L
Linus Torvalds 已提交
1034 1035 1036
	return ret;
}

1037
static inline void print_unload_info(struct seq_file *m, struct module *mod)
L
Linus Torvalds 已提交
1038 1039 1040 1041
{
	struct module_use *use;
	int printed_something = 0;

1042
	seq_printf(m, " %i ", module_refcount(mod));
L
Linus Torvalds 已提交
1043

1044 1045 1046 1047
	/*
	 * Always include a trailing , so userspace can differentiate
	 * between this and the old multi-field proc format.
	 */
1048
	list_for_each_entry(use, &mod->source_list, source_list) {
L
Linus Torvalds 已提交
1049
		printed_something = 1;
1050
		seq_printf(m, "%s,", use->source->name);
L
Linus Torvalds 已提交
1051 1052 1053 1054
	}

	if (mod->init != NULL && mod->exit == NULL) {
		printed_something = 1;
1055
		seq_puts(m, "[permanent],");
L
Linus Torvalds 已提交
1056 1057 1058
	}

	if (!printed_something)
1059
		seq_puts(m, "-");
L
Linus Torvalds 已提交
1060 1061 1062 1063 1064 1065
}

void __symbol_put(const char *symbol)
{
	struct module *owner;

R
Rusty Russell 已提交
1066
	preempt_disable();
1067
	if (!find_symbol(symbol, &owner, NULL, true, false))
L
Linus Torvalds 已提交
1068 1069
		BUG();
	module_put(owner);
R
Rusty Russell 已提交
1070
	preempt_enable();
L
Linus Torvalds 已提交
1071 1072 1073
}
EXPORT_SYMBOL(__symbol_put);

1074
/* Note this assumes addr is a function, which it currently always is. */
L
Linus Torvalds 已提交
1075 1076
void symbol_put_addr(void *addr)
{
1077
	struct module *modaddr;
1078
	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
L
Linus Torvalds 已提交
1079

1080
	if (core_kernel_text(a))
1081
		return;
L
Linus Torvalds 已提交
1082

1083 1084 1085 1086 1087
	/*
	 * Even though we hold a reference on the module; we still need to
	 * disable preemption in order to safely traverse the data structure.
	 */
	preempt_disable();
1088
	modaddr = __module_text_address(a);
1089
	BUG_ON(!modaddr);
1090
	module_put(modaddr);
1091
	preempt_enable();
L
Linus Torvalds 已提交
1092 1093 1094 1095
}
EXPORT_SYMBOL_GPL(symbol_put_addr);

static ssize_t show_refcnt(struct module_attribute *mattr,
1096
			   struct module_kobject *mk, char *buffer)
L
Linus Torvalds 已提交
1097
{
1098
	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
L
Linus Torvalds 已提交
1099 1100
}

1101 1102
static struct module_attribute modinfo_refcnt =
	__ATTR(refcnt, 0444, show_refcnt, NULL);
L
Linus Torvalds 已提交
1103

1104 1105 1106 1107
void __module_get(struct module *module)
{
	if (module) {
		preempt_disable();
1108
		atomic_inc(&module->refcnt);
1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
		trace_module_get(module, _RET_IP_);
		preempt_enable();
	}
}
EXPORT_SYMBOL(__module_get);

bool try_module_get(struct module *module)
{
	bool ret = true;

	if (module) {
		preempt_disable();
1121 1122 1123
		/* Note: here, we can fail to get a reference */
		if (likely(module_is_live(module) &&
			   atomic_inc_not_zero(&module->refcnt) != 0))
1124
			trace_module_get(module, _RET_IP_);
1125
		else
1126 1127 1128 1129 1130 1131 1132 1133
			ret = false;

		preempt_enable();
	}
	return ret;
}
EXPORT_SYMBOL(try_module_get);

A
Al Viro 已提交
1134 1135
void module_put(struct module *module)
{
1136 1137
	int ret;

A
Al Viro 已提交
1138
	if (module) {
1139
		preempt_disable();
1140 1141
		ret = atomic_dec_if_positive(&module->refcnt);
		WARN_ON(ret < 0);	/* Failed to put refcount */
1142
		trace_module_put(module, _RET_IP_);
1143
		preempt_enable();
A
Al Viro 已提交
1144 1145 1146 1147
	}
}
EXPORT_SYMBOL(module_put);

L
Linus Torvalds 已提交
1148
#else /* !CONFIG_MODULE_UNLOAD */
1149
static inline void print_unload_info(struct seq_file *m, struct module *mod)
L
Linus Torvalds 已提交
1150 1151
{
	/* We don't know the usage count, or what modules are using. */
1152
	seq_puts(m, " - -");
L
Linus Torvalds 已提交
1153 1154 1155 1156 1157 1158
}

static inline void module_unload_free(struct module *mod)
{
}

1159
int ref_module(struct module *a, struct module *b)
L
Linus Torvalds 已提交
1160
{
1161
	return strong_try_module_get(b);
L
Linus Torvalds 已提交
1162
}
1163
EXPORT_SYMBOL_GPL(ref_module);
L
Linus Torvalds 已提交
1164

1165
static inline int module_unload_init(struct module *mod)
L
Linus Torvalds 已提交
1166
{
1167
	return 0;
L
Linus Torvalds 已提交
1168 1169 1170
}
#endif /* CONFIG_MODULE_UNLOAD */

1171 1172 1173
static size_t module_flags_taint(struct module *mod, char *buf)
{
	size_t l = 0;
1174 1175 1176 1177
	int i;

	for (i = 0; i < TAINT_FLAGS_COUNT; i++) {
		if (taint_flags[i].module && test_bit(i, &mod->taints))
1178
			buf[l++] = taint_flags[i].c_true;
1179
	}
1180 1181 1182 1183

	return l;
}

1184
static ssize_t show_initstate(struct module_attribute *mattr,
1185
			      struct module_kobject *mk, char *buffer)
1186 1187 1188
{
	const char *state = "unknown";

1189
	switch (mk->mod->state) {
1190 1191 1192 1193 1194 1195 1196 1197 1198
	case MODULE_STATE_LIVE:
		state = "live";
		break;
	case MODULE_STATE_COMING:
		state = "coming";
		break;
	case MODULE_STATE_GOING:
		state = "going";
		break;
1199 1200
	default:
		BUG();
1201 1202 1203 1204
	}
	return sprintf(buffer, "%s\n", state);
}

1205 1206
static struct module_attribute modinfo_initstate =
	__ATTR(initstate, 0444, show_initstate, NULL);
1207

1208 1209 1210 1211
static ssize_t store_uevent(struct module_attribute *mattr,
			    struct module_kobject *mk,
			    const char *buffer, size_t count)
{
1212
	kobject_synth_uevent(&mk->kobj, buffer, count);
1213 1214 1215
	return count;
}

1216 1217 1218 1219 1220 1221
struct module_attribute module_uevent =
	__ATTR(uevent, 0200, NULL, store_uevent);

static ssize_t show_coresize(struct module_attribute *mattr,
			     struct module_kobject *mk, char *buffer)
{
1222
	return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
1223 1224 1225 1226 1227 1228 1229 1230
}

static struct module_attribute modinfo_coresize =
	__ATTR(coresize, 0444, show_coresize, NULL);

static ssize_t show_initsize(struct module_attribute *mattr,
			     struct module_kobject *mk, char *buffer)
{
1231
	return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
}

static struct module_attribute modinfo_initsize =
	__ATTR(initsize, 0444, show_initsize, NULL);

static ssize_t show_taint(struct module_attribute *mattr,
			  struct module_kobject *mk, char *buffer)
{
	size_t l;

	l = module_flags_taint(mk->mod, buffer);
	buffer[l++] = '\n';
	return l;
}

static struct module_attribute modinfo_taint =
	__ATTR(taint, 0444, show_taint, NULL);
1249

1250
static struct module_attribute *modinfo_attrs[] = {
1251
	&module_uevent,
1252 1253
	&modinfo_version,
	&modinfo_srcversion,
1254 1255 1256 1257
	&modinfo_initstate,
	&modinfo_coresize,
	&modinfo_initsize,
	&modinfo_taint,
1258
#ifdef CONFIG_MODULE_UNLOAD
1259
	&modinfo_refcnt,
1260 1261 1262 1263
#endif
	NULL,
};

L
Linus Torvalds 已提交
1264 1265
static const char vermagic[] = VERMAGIC_STRING;

1266
static int try_to_force_load(struct module *mod, const char *reason)
1267 1268
{
#ifdef CONFIG_MODULE_FORCE_LOAD
A
Andi Kleen 已提交
1269
	if (!test_taint(TAINT_FORCED_MODULE))
A
Andrew Morton 已提交
1270
		pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1271
	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1272 1273 1274 1275 1276 1277
	return 0;
#else
	return -ENOEXEC;
#endif
}

L
Linus Torvalds 已提交
1278
#ifdef CONFIG_MODVERSIONS
1279 1280

static u32 resolve_rel_crc(const s32 *crc)
1281
{
1282
	return *(u32 *)((void *)crc + *crc);
1283 1284
}

1285
static int check_version(const struct load_info *info,
L
Linus Torvalds 已提交
1286
			 const char *symname,
1287
			 struct module *mod,
1288
			 const s32 *crc)
L
Linus Torvalds 已提交
1289
{
1290 1291
	Elf_Shdr *sechdrs = info->sechdrs;
	unsigned int versindex = info->index.vers;
L
Linus Torvalds 已提交
1292 1293 1294 1295 1296 1297 1298
	unsigned int i, num_versions;
	struct modversion_info *versions;

	/* Exporting module didn't supply crcs?  OK, we're already tainted. */
	if (!crc)
		return 1;

1299 1300 1301 1302
	/* No versions at all?  modprobe --force does this. */
	if (versindex == 0)
		return try_to_force_load(mod, symname) == 0;

L
Linus Torvalds 已提交
1303 1304 1305 1306 1307
	versions = (void *) sechdrs[versindex].sh_addr;
	num_versions = sechdrs[versindex].sh_size
		/ sizeof(struct modversion_info);

	for (i = 0; i < num_versions; i++) {
1308 1309
		u32 crcval;

L
Linus Torvalds 已提交
1310 1311 1312
		if (strcmp(versions[i].name, symname) != 0)
			continue;

1313 1314 1315 1316 1317
		if (IS_ENABLED(CONFIG_MODULE_REL_CRCS))
			crcval = resolve_rel_crc(crc);
		else
			crcval = *crc;
		if (versions[i].crc == crcval)
L
Linus Torvalds 已提交
1318
			return 1;
1319 1320
		pr_debug("Found checksum %X vs module %lX\n",
			 crcval, versions[i].crc);
1321
		goto bad_version;
L
Linus Torvalds 已提交
1322
	}
1323

1324
	/* Broken toolchain. Warn once, then let it go.. */
K
Kees Cook 已提交
1325
	pr_warn_once("%s: no symbol version for %s\n", info->name, symname);
1326
	return 1;
1327 1328

bad_version:
1329
	pr_warn("%s: disagrees about version of symbol %s\n",
K
Kees Cook 已提交
1330
	       info->name, symname);
1331
	return 0;
L
Linus Torvalds 已提交
1332 1333
}

1334
static inline int check_modstruct_version(const struct load_info *info,
L
Linus Torvalds 已提交
1335 1336
					  struct module *mod)
{
1337
	const s32 *crc;
L
Linus Torvalds 已提交
1338

1339 1340 1341 1342 1343
	/*
	 * Since this should be found in kernel (which can't be removed), no
	 * locking is necessary -- use preempt_disable() to placate lockdep.
	 */
	preempt_disable();
R
Rusty Russell 已提交
1344
	if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL,
1345 1346
			 &crc, true, false)) {
		preempt_enable();
L
Linus Torvalds 已提交
1347
		BUG();
1348 1349
	}
	preempt_enable();
1350 1351
	return check_version(info, VMLINUX_SYMBOL_STR(module_layout),
			     mod, crc);
L
Linus Torvalds 已提交
1352 1353
}

1354 1355 1356
/* First part is kernel version, which we ignore if module has crcs. */
static inline int same_magic(const char *amagic, const char *bmagic,
			     bool has_crcs)
L
Linus Torvalds 已提交
1357
{
1358 1359 1360 1361
	if (has_crcs) {
		amagic += strcspn(amagic, " ");
		bmagic += strcspn(bmagic, " ");
	}
L
Linus Torvalds 已提交
1362 1363 1364
	return strcmp(amagic, bmagic) == 0;
}
#else
1365
static inline int check_version(const struct load_info *info,
L
Linus Torvalds 已提交
1366
				const char *symname,
1367
				struct module *mod,
1368
				const s32 *crc)
L
Linus Torvalds 已提交
1369 1370 1371 1372
{
	return 1;
}

1373
static inline int check_modstruct_version(const struct load_info *info,
L
Linus Torvalds 已提交
1374 1375 1376 1377 1378
					  struct module *mod)
{
	return 1;
}

1379 1380
static inline int same_magic(const char *amagic, const char *bmagic,
			     bool has_crcs)
L
Linus Torvalds 已提交
1381 1382 1383 1384 1385
{
	return strcmp(amagic, bmagic) == 0;
}
#endif /* CONFIG_MODVERSIONS */

1386
/* Resolve a symbol for this module.  I.e. if we find one, record usage. */
1387 1388
static const struct kernel_symbol *resolve_symbol(struct module *mod,
						  const struct load_info *info,
1389
						  const char *name,
1390
						  char ownername[])
L
Linus Torvalds 已提交
1391 1392
{
	struct module *owner;
1393
	const struct kernel_symbol *sym;
1394
	const s32 *crc;
1395
	int err;
L
Linus Torvalds 已提交
1396

1397 1398 1399 1400 1401 1402
	/*
	 * The module_mutex should not be a heavily contended lock;
	 * if we get the occasional sleep here, we'll go an extra iteration
	 * in the wait_event_interruptible(), which is harmless.
	 */
	sched_annotate_sleep();
1403
	mutex_lock(&module_mutex);
1404
	sym = find_symbol(name, &owner, &crc,
A
Andi Kleen 已提交
1405
			  !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
1406 1407 1408
	if (!sym)
		goto unlock;

1409
	if (!check_version(info, name, mod, crc)) {
1410 1411
		sym = ERR_PTR(-EINVAL);
		goto getname;
L
Linus Torvalds 已提交
1412
	}
1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423

	err = ref_module(mod, owner);
	if (err) {
		sym = ERR_PTR(err);
		goto getname;
	}

getname:
	/* We must make copy under the lock if we failed to get ref. */
	strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
unlock:
1424
	mutex_unlock(&module_mutex);
1425
	return sym;
L
Linus Torvalds 已提交
1426 1427
}

1428 1429 1430 1431
static const struct kernel_symbol *
resolve_symbol_wait(struct module *mod,
		    const struct load_info *info,
		    const char *name)
1432 1433
{
	const struct kernel_symbol *ksym;
1434
	char owner[MODULE_NAME_LEN];
1435 1436

	if (wait_event_interruptible_timeout(module_wq,
1437 1438
			!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
			|| PTR_ERR(ksym) != -EBUSY,
1439
					     30 * HZ) <= 0) {
A
Andrew Morton 已提交
1440 1441
		pr_warn("%s: gave up waiting for init of module %s.\n",
			mod->name, owner);
1442 1443 1444 1445
	}
	return ksym;
}

L
Linus Torvalds 已提交
1446 1447 1448 1449
/*
 * /sys/module/foo/sections stuff
 * J. Corbet <corbet@lwn.net>
 */
R
Rusty Russell 已提交
1450
#ifdef CONFIG_SYSFS
1451

R
Rusty Russell 已提交
1452
#ifdef CONFIG_KALLSYMS
1453 1454 1455 1456 1457
static inline bool sect_empty(const Elf_Shdr *sect)
{
	return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
}

1458
struct module_sect_attr {
1459 1460 1461 1462 1463
	struct module_attribute mattr;
	char *name;
	unsigned long address;
};

1464
struct module_sect_attrs {
1465 1466 1467 1468 1469
	struct attribute_group grp;
	unsigned int nsections;
	struct module_sect_attr attrs[0];
};

L
Linus Torvalds 已提交
1470
static ssize_t module_sect_show(struct module_attribute *mattr,
1471
				struct module_kobject *mk, char *buf)
L
Linus Torvalds 已提交
1472 1473 1474
{
	struct module_sect_attr *sattr =
		container_of(mattr, struct module_sect_attr, mattr);
1475
	return sprintf(buf, "0x%pK\n", (void *)sattr->address);
L
Linus Torvalds 已提交
1476 1477
}

1478 1479
static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
{
1480
	unsigned int section;
1481 1482 1483 1484 1485 1486

	for (section = 0; section < sect_attrs->nsections; section++)
		kfree(sect_attrs->attrs[section].name);
	kfree(sect_attrs);
}

R
Rusty Russell 已提交
1487
static void add_sect_attrs(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
1488 1489 1490 1491 1492
{
	unsigned int nloaded = 0, i, size[2];
	struct module_sect_attrs *sect_attrs;
	struct module_sect_attr *sattr;
	struct attribute **gattr;
D
Daniel Walker 已提交
1493

L
Linus Torvalds 已提交
1494
	/* Count loaded sections and allocate structures */
R
Rusty Russell 已提交
1495 1496
	for (i = 0; i < info->hdr->e_shnum; i++)
		if (!sect_empty(&info->sechdrs[i]))
L
Linus Torvalds 已提交
1497 1498 1499 1500 1501
			nloaded++;
	size[0] = ALIGN(sizeof(*sect_attrs)
			+ nloaded * sizeof(sect_attrs->attrs[0]),
			sizeof(sect_attrs->grp.attrs[0]));
	size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.attrs[0]);
1502 1503
	sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL);
	if (sect_attrs == NULL)
L
Linus Torvalds 已提交
1504 1505 1506 1507 1508 1509
		return;

	/* Setup section attributes. */
	sect_attrs->grp.name = "sections";
	sect_attrs->grp.attrs = (void *)sect_attrs + size[0];

1510
	sect_attrs->nsections = 0;
L
Linus Torvalds 已提交
1511 1512
	sattr = &sect_attrs->attrs[0];
	gattr = &sect_attrs->grp.attrs[0];
R
Rusty Russell 已提交
1513 1514 1515
	for (i = 0; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *sec = &info->sechdrs[i];
		if (sect_empty(sec))
1516
			continue;
R
Rusty Russell 已提交
1517 1518
		sattr->address = sec->sh_addr;
		sattr->name = kstrdup(info->secstrings + sec->sh_name,
1519 1520 1521 1522
					GFP_KERNEL);
		if (sattr->name == NULL)
			goto out;
		sect_attrs->nsections++;
1523
		sysfs_attr_init(&sattr->mattr.attr);
L
Linus Torvalds 已提交
1524 1525 1526
		sattr->mattr.show = module_sect_show;
		sattr->mattr.store = NULL;
		sattr->mattr.attr.name = sattr->name;
1527
		sattr->mattr.attr.mode = S_IRUSR;
L
Linus Torvalds 已提交
1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
		*(gattr++) = &(sattr++)->mattr.attr;
	}
	*gattr = NULL;

	if (sysfs_create_group(&mod->mkobj.kobj, &sect_attrs->grp))
		goto out;

	mod->sect_attrs = sect_attrs;
	return;
  out:
1538
	free_sect_attrs(sect_attrs);
L
Linus Torvalds 已提交
1539 1540 1541 1542 1543 1544 1545 1546 1547
}

static void remove_sect_attrs(struct module *mod)
{
	if (mod->sect_attrs) {
		sysfs_remove_group(&mod->mkobj.kobj,
				   &mod->sect_attrs->grp);
		/* We are positive that no one is using any sect attrs
		 * at this point.  Deallocate immediately. */
1548
		free_sect_attrs(mod->sect_attrs);
L
Linus Torvalds 已提交
1549 1550 1551 1552
		mod->sect_attrs = NULL;
	}
}

R
Roland McGrath 已提交
1553 1554 1555 1556 1557 1558 1559 1560 1561 1562
/*
 * /sys/module/foo/notes/.section.name gives contents of SHT_NOTE sections.
 */

struct module_notes_attrs {
	struct kobject *dir;
	unsigned int notes;
	struct bin_attribute attrs[0];
};

1563
static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
R
Roland McGrath 已提交
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
				 struct bin_attribute *bin_attr,
				 char *buf, loff_t pos, size_t count)
{
	/*
	 * The caller checked the pos and count against our size.
	 */
	memcpy(buf, bin_attr->private + pos, count);
	return count;
}

static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
			     unsigned int i)
{
	if (notes_attrs->dir) {
		while (i-- > 0)
			sysfs_remove_bin_file(notes_attrs->dir,
					      &notes_attrs->attrs[i]);
1581
		kobject_put(notes_attrs->dir);
R
Roland McGrath 已提交
1582 1583 1584 1585
	}
	kfree(notes_attrs);
}

R
Rusty Russell 已提交
1586
static void add_notes_attrs(struct module *mod, const struct load_info *info)
R
Roland McGrath 已提交
1587 1588 1589 1590 1591
{
	unsigned int notes, loaded, i;
	struct module_notes_attrs *notes_attrs;
	struct bin_attribute *nattr;

1592 1593 1594 1595
	/* failed to create section attributes, so can't create notes */
	if (!mod->sect_attrs)
		return;

R
Roland McGrath 已提交
1596 1597
	/* Count notes sections and allocate structures.  */
	notes = 0;
R
Rusty Russell 已提交
1598 1599 1600
	for (i = 0; i < info->hdr->e_shnum; i++)
		if (!sect_empty(&info->sechdrs[i]) &&
		    (info->sechdrs[i].sh_type == SHT_NOTE))
R
Roland McGrath 已提交
1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613
			++notes;

	if (notes == 0)
		return;

	notes_attrs = kzalloc(sizeof(*notes_attrs)
			      + notes * sizeof(notes_attrs->attrs[0]),
			      GFP_KERNEL);
	if (notes_attrs == NULL)
		return;

	notes_attrs->notes = notes;
	nattr = &notes_attrs->attrs[0];
R
Rusty Russell 已提交
1614 1615
	for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
		if (sect_empty(&info->sechdrs[i]))
R
Roland McGrath 已提交
1616
			continue;
R
Rusty Russell 已提交
1617
		if (info->sechdrs[i].sh_type == SHT_NOTE) {
1618
			sysfs_bin_attr_init(nattr);
R
Roland McGrath 已提交
1619 1620
			nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
			nattr->attr.mode = S_IRUGO;
R
Rusty Russell 已提交
1621 1622
			nattr->size = info->sechdrs[i].sh_size;
			nattr->private = (void *) info->sechdrs[i].sh_addr;
R
Roland McGrath 已提交
1623 1624 1625 1626 1627 1628
			nattr->read = module_notes_read;
			++nattr;
		}
		++loaded;
	}

1629
	notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
R
Roland McGrath 已提交
1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
	if (!notes_attrs->dir)
		goto out;

	for (i = 0; i < notes; ++i)
		if (sysfs_create_bin_file(notes_attrs->dir,
					  &notes_attrs->attrs[i]))
			goto out;

	mod->notes_attrs = notes_attrs;
	return;

  out:
	free_notes_attrs(notes_attrs, i);
}

static void remove_notes_attrs(struct module *mod)
{
	if (mod->notes_attrs)
		free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
}

L
Linus Torvalds 已提交
1651
#else
1652

R
Rusty Russell 已提交
1653 1654
static inline void add_sect_attrs(struct module *mod,
				  const struct load_info *info)
L
Linus Torvalds 已提交
1655 1656 1657 1658 1659 1660
{
}

static inline void remove_sect_attrs(struct module *mod)
{
}
R
Roland McGrath 已提交
1661

R
Rusty Russell 已提交
1662 1663
static inline void add_notes_attrs(struct module *mod,
				   const struct load_info *info)
R
Roland McGrath 已提交
1664 1665 1666 1667 1668 1669
{
}

static inline void remove_notes_attrs(struct module *mod)
{
}
R
Rusty Russell 已提交
1670
#endif /* CONFIG_KALLSYMS */
L
Linus Torvalds 已提交
1671

1672
static void del_usage_links(struct module *mod)
1673 1674 1675 1676
{
#ifdef CONFIG_MODULE_UNLOAD
	struct module_use *use;

1677
	mutex_lock(&module_mutex);
1678 1679
	list_for_each_entry(use, &mod->target_list, target_list)
		sysfs_remove_link(use->target->holders_dir, mod->name);
1680
	mutex_unlock(&module_mutex);
1681 1682 1683
#endif
}

1684
static int add_usage_links(struct module *mod)
1685
{
1686
	int ret = 0;
1687 1688 1689
#ifdef CONFIG_MODULE_UNLOAD
	struct module_use *use;

1690
	mutex_lock(&module_mutex);
1691 1692 1693 1694 1695 1696
	list_for_each_entry(use, &mod->target_list, target_list) {
		ret = sysfs_create_link(use->target->holders_dir,
					&mod->mkobj.kobj, mod->name);
		if (ret)
			break;
	}
1697
	mutex_unlock(&module_mutex);
1698 1699
	if (ret)
		del_usage_links(mod);
1700
#endif
1701
	return ret;
1702 1703
}

1704
static int module_add_modinfo_attrs(struct module *mod)
1705 1706
{
	struct module_attribute *attr;
1707
	struct module_attribute *temp_attr;
1708 1709 1710
	int error = 0;
	int i;

1711 1712 1713 1714 1715 1716 1717
	mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
					(ARRAY_SIZE(modinfo_attrs) + 1)),
					GFP_KERNEL);
	if (!mod->modinfo_attrs)
		return -ENOMEM;

	temp_attr = mod->modinfo_attrs;
1718
	for (i = 0; (attr = modinfo_attrs[i]) && !error; i++) {
R
Rusty Russell 已提交
1719
		if (!attr->test || attr->test(mod)) {
1720
			memcpy(temp_attr, attr, sizeof(*temp_attr));
1721
			sysfs_attr_init(&temp_attr->attr);
1722 1723
			error = sysfs_create_file(&mod->mkobj.kobj,
					&temp_attr->attr);
1724 1725
			++temp_attr;
		}
1726 1727 1728 1729
	}
	return error;
}

1730
static void module_remove_modinfo_attrs(struct module *mod)
1731 1732 1733 1734
{
	struct module_attribute *attr;
	int i;

1735 1736 1737 1738
	for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
		/* pick a field to test for end of list */
		if (!attr->attr.name)
			break;
1739
		sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
1740 1741
		if (attr->free)
			attr->free(mod);
1742
	}
1743
	kfree(mod->modinfo_attrs);
1744
}
L
Linus Torvalds 已提交
1745

1746 1747 1748 1749 1750 1751 1752 1753
static void mod_kobject_put(struct module *mod)
{
	DECLARE_COMPLETION_ONSTACK(c);
	mod->mkobj.kobj_completion = &c;
	kobject_put(&mod->mkobj.kobj);
	wait_for_completion(&c);
}

1754
static int mod_sysfs_init(struct module *mod)
L
Linus Torvalds 已提交
1755 1756
{
	int err;
1757
	struct kobject *kobj;
L
Linus Torvalds 已提交
1758

1759
	if (!module_sysfs_initialized) {
A
Andrew Morton 已提交
1760
		pr_err("%s: module sysfs not initialized\n", mod->name);
1761 1762 1763
		err = -EINVAL;
		goto out;
	}
1764 1765 1766

	kobj = kset_find_obj(module_kset, mod->name);
	if (kobj) {
A
Andrew Morton 已提交
1767
		pr_err("%s: module is already loaded\n", mod->name);
1768 1769 1770 1771 1772
		kobject_put(kobj);
		err = -EINVAL;
		goto out;
	}

L
Linus Torvalds 已提交
1773
	mod->mkobj.mod = mod;
1774

1775 1776 1777 1778 1779
	memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
	mod->mkobj.kobj.kset = module_kset;
	err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
				   "%s", mod->name);
	if (err)
1780
		mod_kobject_put(mod);
K
Kay Sievers 已提交
1781

1782
	/* delay uevent until full sysfs population */
K
Kay Sievers 已提交
1783 1784 1785 1786
out:
	return err;
}

1787
static int mod_sysfs_setup(struct module *mod,
R
Rusty Russell 已提交
1788
			   const struct load_info *info,
K
Kay Sievers 已提交
1789 1790 1791 1792 1793
			   struct kernel_param *kparam,
			   unsigned int num_params)
{
	int err;

1794 1795 1796 1797
	err = mod_sysfs_init(mod);
	if (err)
		goto out;

1798
	mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
1799 1800
	if (!mod->holders_dir) {
		err = -ENOMEM;
K
Kay Sievers 已提交
1801
		goto out_unreg;
1802
	}
K
Kay Sievers 已提交
1803

L
Linus Torvalds 已提交
1804 1805
	err = module_param_sysfs_setup(mod, kparam, num_params);
	if (err)
K
Kay Sievers 已提交
1806
		goto out_unreg_holders;
L
Linus Torvalds 已提交
1807

1808 1809
	err = module_add_modinfo_attrs(mod);
	if (err)
1810
		goto out_unreg_param;
1811

1812 1813 1814 1815
	err = add_usage_links(mod);
	if (err)
		goto out_unreg_modinfo_attrs;

R
Rusty Russell 已提交
1816 1817
	add_sect_attrs(mod, info);
	add_notes_attrs(mod, info);
1818

1819
	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
L
Linus Torvalds 已提交
1820 1821
	return 0;

1822 1823
out_unreg_modinfo_attrs:
	module_remove_modinfo_attrs(mod);
1824 1825
out_unreg_param:
	module_param_sysfs_remove(mod);
K
Kay Sievers 已提交
1826
out_unreg_holders:
1827
	kobject_put(mod->holders_dir);
K
Kay Sievers 已提交
1828
out_unreg:
1829
	mod_kobject_put(mod);
1830
out:
L
Linus Torvalds 已提交
1831 1832
	return err;
}
1833 1834 1835

static void mod_sysfs_fini(struct module *mod)
{
R
Rusty Russell 已提交
1836 1837
	remove_notes_attrs(mod);
	remove_sect_attrs(mod);
1838
	mod_kobject_put(mod);
1839 1840
}

1841 1842 1843 1844
static void init_param_lock(struct module *mod)
{
	mutex_init(&mod->param_lock);
}
R
Rusty Russell 已提交
1845
#else /* !CONFIG_SYSFS */
1846

R
Rusty Russell 已提交
1847 1848
static int mod_sysfs_setup(struct module *mod,
			   const struct load_info *info,
1849 1850 1851 1852 1853 1854
			   struct kernel_param *kparam,
			   unsigned int num_params)
{
	return 0;
}

1855 1856 1857 1858
static void mod_sysfs_fini(struct module *mod)
{
}

1859 1860 1861 1862
static void module_remove_modinfo_attrs(struct module *mod)
{
}

1863 1864 1865 1866
static void del_usage_links(struct module *mod)
{
}

1867 1868 1869
static void init_param_lock(struct module *mod)
{
}
1870
#endif /* CONFIG_SYSFS */
L
Linus Torvalds 已提交
1871

1872
static void mod_sysfs_teardown(struct module *mod)
L
Linus Torvalds 已提交
1873
{
1874
	del_usage_links(mod);
1875
	module_remove_modinfo_attrs(mod);
L
Linus Torvalds 已提交
1876
	module_param_sysfs_remove(mod);
1877 1878
	kobject_put(mod->mkobj.drivers_dir);
	kobject_put(mod->holders_dir);
1879
	mod_sysfs_fini(mod);
L
Linus Torvalds 已提交
1880 1881
}

1882
#ifdef CONFIG_STRICT_MODULE_RWX
1883 1884 1885
/*
 * LKM RO/NX protection: protect module's text/ro-data
 * from modification and any data from execution.
R
Rusty Russell 已提交
1886 1887
 *
 * General layout of module is:
J
Jessica Yu 已提交
1888 1889 1890 1891 1892
 *          [text] [read-only-data] [ro-after-init] [writable data]
 * text_size -----^                ^               ^               ^
 * ro_size ------------------------|               |               |
 * ro_after_init_size -----------------------------|               |
 * size -----------------------------------------------------------|
R
Rusty Russell 已提交
1893 1894
 *
 * These values are always page-aligned (as is base)
1895
 */
R
Rusty Russell 已提交
1896 1897
static void frob_text(const struct module_layout *layout,
		      int (*set_memory)(unsigned long start, int num_pages))
1898
{
R
Rusty Russell 已提交
1899 1900 1901 1902
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base,
		   layout->text_size >> PAGE_SHIFT);
1903 1904
}

R
Rusty Russell 已提交
1905 1906
static void frob_rodata(const struct module_layout *layout,
			int (*set_memory)(unsigned long start, int num_pages))
1907
{
R
Rusty Russell 已提交
1908 1909 1910 1911 1912
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base + layout->text_size,
		   (layout->ro_size - layout->text_size) >> PAGE_SHIFT);
1913 1914
}

J
Jessica Yu 已提交
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
static void frob_ro_after_init(const struct module_layout *layout,
				int (*set_memory)(unsigned long start, int num_pages))
{
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->ro_after_init_size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base + layout->ro_size,
		   (layout->ro_after_init_size - layout->ro_size) >> PAGE_SHIFT);
}

R
Rusty Russell 已提交
1925 1926
static void frob_writable_data(const struct module_layout *layout,
			       int (*set_memory)(unsigned long start, int num_pages))
1927
{
R
Rusty Russell 已提交
1928
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
J
Jessica Yu 已提交
1929
	BUG_ON((unsigned long)layout->ro_after_init_size & (PAGE_SIZE-1));
R
Rusty Russell 已提交
1930
	BUG_ON((unsigned long)layout->size & (PAGE_SIZE-1));
J
Jessica Yu 已提交
1931 1932
	set_memory((unsigned long)layout->base + layout->ro_after_init_size,
		   (layout->size - layout->ro_after_init_size) >> PAGE_SHIFT);
1933 1934
}

R
Rusty Russell 已提交
1935 1936
/* livepatching wants to disable read-only so it can frob module. */
void module_disable_ro(const struct module *mod)
1937
{
1938 1939 1940
	if (!rodata_enabled)
		return;

R
Rusty Russell 已提交
1941 1942
	frob_text(&mod->core_layout, set_memory_rw);
	frob_rodata(&mod->core_layout, set_memory_rw);
J
Jessica Yu 已提交
1943
	frob_ro_after_init(&mod->core_layout, set_memory_rw);
R
Rusty Russell 已提交
1944 1945
	frob_text(&mod->init_layout, set_memory_rw);
	frob_rodata(&mod->init_layout, set_memory_rw);
1946
}
1947

J
Jessica Yu 已提交
1948
void module_enable_ro(const struct module *mod, bool after_init)
1949
{
1950 1951 1952
	if (!rodata_enabled)
		return;

R
Rusty Russell 已提交
1953 1954 1955 1956
	frob_text(&mod->core_layout, set_memory_ro);
	frob_rodata(&mod->core_layout, set_memory_ro);
	frob_text(&mod->init_layout, set_memory_ro);
	frob_rodata(&mod->init_layout, set_memory_ro);
J
Jessica Yu 已提交
1957 1958 1959

	if (after_init)
		frob_ro_after_init(&mod->core_layout, set_memory_ro);
1960 1961
}

R
Rusty Russell 已提交
1962
static void module_enable_nx(const struct module *mod)
1963
{
R
Rusty Russell 已提交
1964
	frob_rodata(&mod->core_layout, set_memory_nx);
J
Jessica Yu 已提交
1965
	frob_ro_after_init(&mod->core_layout, set_memory_nx);
R
Rusty Russell 已提交
1966 1967 1968
	frob_writable_data(&mod->core_layout, set_memory_nx);
	frob_rodata(&mod->init_layout, set_memory_nx);
	frob_writable_data(&mod->init_layout, set_memory_nx);
1969 1970
}

R
Rusty Russell 已提交
1971
static void module_disable_nx(const struct module *mod)
1972
{
R
Rusty Russell 已提交
1973
	frob_rodata(&mod->core_layout, set_memory_x);
J
Jessica Yu 已提交
1974
	frob_ro_after_init(&mod->core_layout, set_memory_x);
R
Rusty Russell 已提交
1975 1976 1977
	frob_writable_data(&mod->core_layout, set_memory_x);
	frob_rodata(&mod->init_layout, set_memory_x);
	frob_writable_data(&mod->init_layout, set_memory_x);
1978 1979 1980
}

/* Iterate through all modules and set each module's text as RW */
1981
void set_all_modules_text_rw(void)
1982 1983 1984
{
	struct module *mod;

1985 1986 1987
	if (!rodata_enabled)
		return;

1988 1989
	mutex_lock(&module_mutex);
	list_for_each_entry_rcu(mod, &modules, list) {
1990 1991
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
R
Rusty Russell 已提交
1992 1993 1994

		frob_text(&mod->core_layout, set_memory_rw);
		frob_text(&mod->init_layout, set_memory_rw);
1995 1996 1997 1998 1999
	}
	mutex_unlock(&module_mutex);
}

/* Iterate through all modules and set each module's text as RO */
2000
void set_all_modules_text_ro(void)
2001 2002 2003
{
	struct module *mod;

2004 2005 2006
	if (!rodata_enabled)
		return;

2007 2008
	mutex_lock(&module_mutex);
	list_for_each_entry_rcu(mod, &modules, list) {
2009 2010 2011 2012 2013 2014 2015
		/*
		 * Ignore going modules since it's possible that ro
		 * protection has already been disabled, otherwise we'll
		 * run into protection faults at module deallocation.
		 */
		if (mod->state == MODULE_STATE_UNFORMED ||
			mod->state == MODULE_STATE_GOING)
2016
			continue;
R
Rusty Russell 已提交
2017 2018 2019

		frob_text(&mod->core_layout, set_memory_ro);
		frob_text(&mod->init_layout, set_memory_ro);
2020 2021 2022
	}
	mutex_unlock(&module_mutex);
}
R
Rusty Russell 已提交
2023 2024 2025

static void disable_ro_nx(const struct module_layout *layout)
{
2026 2027 2028 2029 2030
	if (rodata_enabled) {
		frob_text(layout, set_memory_rw);
		frob_rodata(layout, set_memory_rw);
		frob_ro_after_init(layout, set_memory_rw);
	}
R
Rusty Russell 已提交
2031
	frob_rodata(layout, set_memory_x);
J
Jessica Yu 已提交
2032
	frob_ro_after_init(layout, set_memory_x);
R
Rusty Russell 已提交
2033 2034 2035
	frob_writable_data(layout, set_memory_x);
}

2036
#else
R
Rusty Russell 已提交
2037 2038 2039
static void disable_ro_nx(const struct module_layout *layout) { }
static void module_enable_nx(const struct module *mod) { }
static void module_disable_nx(const struct module *mod) { }
2040 2041
#endif

2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118
#ifdef CONFIG_LIVEPATCH
/*
 * Persist Elf information about a module. Copy the Elf header,
 * section header table, section string table, and symtab section
 * index from info to mod->klp_info.
 */
static int copy_module_elf(struct module *mod, struct load_info *info)
{
	unsigned int size, symndx;
	int ret;

	size = sizeof(*mod->klp_info);
	mod->klp_info = kmalloc(size, GFP_KERNEL);
	if (mod->klp_info == NULL)
		return -ENOMEM;

	/* Elf header */
	size = sizeof(mod->klp_info->hdr);
	memcpy(&mod->klp_info->hdr, info->hdr, size);

	/* Elf section header table */
	size = sizeof(*info->sechdrs) * info->hdr->e_shnum;
	mod->klp_info->sechdrs = kmalloc(size, GFP_KERNEL);
	if (mod->klp_info->sechdrs == NULL) {
		ret = -ENOMEM;
		goto free_info;
	}
	memcpy(mod->klp_info->sechdrs, info->sechdrs, size);

	/* Elf section name string table */
	size = info->sechdrs[info->hdr->e_shstrndx].sh_size;
	mod->klp_info->secstrings = kmalloc(size, GFP_KERNEL);
	if (mod->klp_info->secstrings == NULL) {
		ret = -ENOMEM;
		goto free_sechdrs;
	}
	memcpy(mod->klp_info->secstrings, info->secstrings, size);

	/* Elf symbol section index */
	symndx = info->index.sym;
	mod->klp_info->symndx = symndx;

	/*
	 * For livepatch modules, core_kallsyms.symtab is a complete
	 * copy of the original symbol table. Adjust sh_addr to point
	 * to core_kallsyms.symtab since the copy of the symtab in module
	 * init memory is freed at the end of do_init_module().
	 */
	mod->klp_info->sechdrs[symndx].sh_addr = \
		(unsigned long) mod->core_kallsyms.symtab;

	return 0;

free_sechdrs:
	kfree(mod->klp_info->sechdrs);
free_info:
	kfree(mod->klp_info);
	return ret;
}

static void free_module_elf(struct module *mod)
{
	kfree(mod->klp_info->sechdrs);
	kfree(mod->klp_info->secstrings);
	kfree(mod->klp_info);
}
#else /* !CONFIG_LIVEPATCH */
static int copy_module_elf(struct module *mod, struct load_info *info)
{
	return 0;
}

static void free_module_elf(struct module *mod)
{
}
#endif /* CONFIG_LIVEPATCH */

2119
void __weak module_memfree(void *module_region)
2120 2121 2122 2123 2124 2125 2126 2127
{
	vfree(module_region);
}

void __weak module_arch_cleanup(struct module *mod)
{
}

2128 2129 2130 2131
void __weak module_arch_freeing_init(struct module *mod)
{
}

2132
/* Free a module, remove from lists, etc. */
L
Linus Torvalds 已提交
2133 2134
static void free_module(struct module *mod)
{
2135 2136
	trace_module_free(mod);

2137
	mod_sysfs_teardown(mod);
L
Linus Torvalds 已提交
2138

2139 2140
	/* We leave it in list to prevent duplicate loads, but make sure
	 * that noone uses it while it's being deconstructed. */
2141
	mutex_lock(&module_mutex);
2142
	mod->state = MODULE_STATE_UNFORMED;
2143
	mutex_unlock(&module_mutex);
2144

2145 2146 2147
	/* Remove dynamic debug info */
	ddebug_remove_module(mod->name);

L
Linus Torvalds 已提交
2148 2149 2150 2151 2152 2153
	/* Arch-specific cleanup. */
	module_arch_cleanup(mod);

	/* Module unload stuff */
	module_unload_free(mod);

2154 2155 2156
	/* Free any allocated parameters. */
	destroy_params(mod->kp, mod->num_kp);

2157 2158 2159
	if (is_livepatch_module(mod))
		free_module_elf(mod);

2160 2161
	/* Now we can delete it from the lists */
	mutex_lock(&module_mutex);
2162 2163
	/* Unlink carefully: kallsyms could be walking list. */
	list_del_rcu(&mod->list);
2164
	mod_tree_remove(mod);
2165
	/* Remove this module from bug list, this uses list_del_rcu */
2166
	module_bug_cleanup(mod);
2167 2168
	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
	synchronize_sched();
2169 2170
	mutex_unlock(&module_mutex);

R
Rusty Russell 已提交
2171 2172
	/* This may be empty, but that's OK */
	disable_ro_nx(&mod->init_layout);
2173
	module_arch_freeing_init(mod);
2174
	module_memfree(mod->init_layout.base);
L
Linus Torvalds 已提交
2175
	kfree(mod->args);
2176
	percpu_modfree(mod);
2177

2178
	/* Free lock-classes; relies on the preceding sync_rcu(). */
2179
	lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
I
Ingo Molnar 已提交
2180

L
Linus Torvalds 已提交
2181
	/* Finally, free the core (containing the module structure) */
R
Rusty Russell 已提交
2182
	disable_ro_nx(&mod->core_layout);
2183
	module_memfree(mod->core_layout.base);
2184 2185 2186 2187

#ifdef CONFIG_MPU
	update_protections(current->mm);
#endif
L
Linus Torvalds 已提交
2188 2189 2190 2191 2192
}

void *__symbol_get(const char *symbol)
{
	struct module *owner;
2193
	const struct kernel_symbol *sym;
L
Linus Torvalds 已提交
2194

R
Rusty Russell 已提交
2195
	preempt_disable();
2196 2197 2198
	sym = find_symbol(symbol, &owner, NULL, true, true);
	if (sym && strong_try_module_get(owner))
		sym = NULL;
R
Rusty Russell 已提交
2199
	preempt_enable();
L
Linus Torvalds 已提交
2200

2201
	return sym ? (void *)sym->value : NULL;
L
Linus Torvalds 已提交
2202 2203 2204
}
EXPORT_SYMBOL_GPL(__symbol_get);

2205 2206
/*
 * Ensure that an exported symbol [global namespace] does not already exist
2207
 * in the kernel or in some other module's exported symbol table.
2208 2209
 *
 * You must hold the module_mutex.
2210 2211 2212
 */
static int verify_export_symbols(struct module *mod)
{
2213
	unsigned int i;
2214
	struct module *owner;
2215 2216 2217 2218 2219 2220 2221 2222
	const struct kernel_symbol *s;
	struct {
		const struct kernel_symbol *sym;
		unsigned int num;
	} arr[] = {
		{ mod->syms, mod->num_syms },
		{ mod->gpl_syms, mod->num_gpl_syms },
		{ mod->gpl_future_syms, mod->num_gpl_future_syms },
2223
#ifdef CONFIG_UNUSED_SYMBOLS
2224 2225
		{ mod->unused_syms, mod->num_unused_syms },
		{ mod->unused_gpl_syms, mod->num_unused_gpl_syms },
2226
#endif
2227
	};
2228

2229 2230
	for (i = 0; i < ARRAY_SIZE(arr); i++) {
		for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
2231
			if (find_symbol(s->name, &owner, NULL, true, false)) {
A
Andrew Morton 已提交
2232
				pr_err("%s: exports duplicate symbol %s"
2233 2234 2235 2236
				       " (owned by %s)\n",
				       mod->name, s->name, module_name(owner));
				return -ENOEXEC;
			}
2237
		}
2238 2239
	}
	return 0;
2240 2241
}

2242
/* Change all symbols so that st_value encodes the pointer directly. */
2243 2244 2245 2246
static int simplify_symbols(struct module *mod, const struct load_info *info)
{
	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
	Elf_Sym *sym = (void *)symsec->sh_addr;
L
Linus Torvalds 已提交
2247
	unsigned long secbase;
2248
	unsigned int i;
L
Linus Torvalds 已提交
2249
	int ret = 0;
2250
	const struct kernel_symbol *ksym;
L
Linus Torvalds 已提交
2251

2252 2253 2254
	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
		const char *name = info->strtab + sym[i].st_name;

L
Linus Torvalds 已提交
2255 2256
		switch (sym[i].st_shndx) {
		case SHN_COMMON:
2257 2258 2259 2260
			/* Ignore common symbols */
			if (!strncmp(name, "__gnu_lto", 9))
				break;

L
Linus Torvalds 已提交
2261 2262
			/* We compiled with -fno-common.  These are not
			   supposed to happen.  */
2263
			pr_debug("Common symbol: %s\n", name);
2264
			pr_warn("%s: please compile with -fno-common\n",
L
Linus Torvalds 已提交
2265 2266 2267 2268 2269 2270
			       mod->name);
			ret = -ENOEXEC;
			break;

		case SHN_ABS:
			/* Don't need to do anything */
2271
			pr_debug("Absolute symbol: 0x%08lx\n",
L
Linus Torvalds 已提交
2272 2273 2274
			       (long)sym[i].st_value);
			break;

2275 2276 2277 2278
		case SHN_LIVEPATCH:
			/* Livepatch symbols are resolved by livepatch */
			break;

L
Linus Torvalds 已提交
2279
		case SHN_UNDEF:
2280
			ksym = resolve_symbol_wait(mod, info, name);
L
Linus Torvalds 已提交
2281
			/* Ok if resolved.  */
2282
			if (ksym && !IS_ERR(ksym)) {
2283
				sym[i].st_value = ksym->value;
L
Linus Torvalds 已提交
2284
				break;
2285 2286
			}

L
Linus Torvalds 已提交
2287
			/* Ok if weak.  */
2288
			if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
L
Linus Torvalds 已提交
2289 2290
				break;

A
Andrew Morton 已提交
2291 2292
			pr_warn("%s: Unknown symbol %s (err %li)\n",
				mod->name, name, PTR_ERR(ksym));
2293
			ret = PTR_ERR(ksym) ?: -ENOENT;
L
Linus Torvalds 已提交
2294 2295 2296 2297
			break;

		default:
			/* Divert to percpu allocation if a percpu var. */
2298
			if (sym[i].st_shndx == info->index.pcpu)
2299
				secbase = (unsigned long)mod_percpu(mod);
L
Linus Torvalds 已提交
2300
			else
2301
				secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
L
Linus Torvalds 已提交
2302 2303 2304 2305 2306 2307 2308 2309
			sym[i].st_value += secbase;
			break;
		}
	}

	return ret;
}

2310
static int apply_relocations(struct module *mod, const struct load_info *info)
2311 2312 2313 2314 2315
{
	unsigned int i;
	int err = 0;

	/* Now do relocations. */
2316 2317
	for (i = 1; i < info->hdr->e_shnum; i++) {
		unsigned int infosec = info->sechdrs[i].sh_info;
2318 2319

		/* Not a valid relocation section? */
2320
		if (infosec >= info->hdr->e_shnum)
2321 2322 2323
			continue;

		/* Don't bother with non-allocated sections */
2324
		if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
2325 2326
			continue;

2327 2328 2329 2330
		/* Livepatch relocation sections are applied by livepatch */
		if (info->sechdrs[i].sh_flags & SHF_RELA_LIVEPATCH)
			continue;

2331 2332 2333 2334 2335 2336
		if (info->sechdrs[i].sh_type == SHT_REL)
			err = apply_relocate(info->sechdrs, info->strtab,
					     info->index.sym, i, mod);
		else if (info->sechdrs[i].sh_type == SHT_RELA)
			err = apply_relocate_add(info->sechdrs, info->strtab,
						 info->index.sym, i, mod);
2337 2338 2339 2340 2341 2342
		if (err < 0)
			break;
	}
	return err;
}

2343 2344 2345 2346 2347 2348 2349 2350
/* Additional bytes needed by arch in front of individual sections */
unsigned int __weak arch_mod_section_prepend(struct module *mod,
					     unsigned int section)
{
	/* default implementation just returns zero */
	return 0;
}

L
Linus Torvalds 已提交
2351
/* Update size with this section: return offset. */
2352 2353
static long get_offset(struct module *mod, unsigned int *size,
		       Elf_Shdr *sechdr, unsigned int section)
L
Linus Torvalds 已提交
2354 2355 2356
{
	long ret;

2357
	*size += arch_mod_section_prepend(mod, section);
L
Linus Torvalds 已提交
2358 2359 2360 2361 2362 2363 2364 2365 2366
	ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
	*size = ret + sechdr->sh_size;
	return ret;
}

/* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
   might -- code, read-only data, read-write data, small data.  Tally
   sizes, and place the offsets into sh_entsize fields: high bit means it
   belongs in init. */
2367
static void layout_sections(struct module *mod, struct load_info *info)
L
Linus Torvalds 已提交
2368 2369 2370 2371 2372 2373 2374
{
	static unsigned long const masks[][2] = {
		/* NOTE: all executable code must be the first section
		 * in this array; otherwise modify the text_size
		 * finder in the two loops below */
		{ SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
		{ SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
J
Jessica Yu 已提交
2375
		{ SHF_RO_AFTER_INIT | SHF_ALLOC, ARCH_SHF_SMALL },
L
Linus Torvalds 已提交
2376 2377 2378 2379 2380
		{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
		{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
	};
	unsigned int m, i;

2381 2382
	for (i = 0; i < info->hdr->e_shnum; i++)
		info->sechdrs[i].sh_entsize = ~0UL;
L
Linus Torvalds 已提交
2383

2384
	pr_debug("Core section allocation order:\n");
L
Linus Torvalds 已提交
2385
	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
2386 2387 2388
		for (i = 0; i < info->hdr->e_shnum; ++i) {
			Elf_Shdr *s = &info->sechdrs[i];
			const char *sname = info->secstrings + s->sh_name;
L
Linus Torvalds 已提交
2389 2390 2391 2392

			if ((s->sh_flags & masks[m][0]) != masks[m][0]
			    || (s->sh_flags & masks[m][1])
			    || s->sh_entsize != ~0UL
2393
			    || strstarts(sname, ".init"))
L
Linus Torvalds 已提交
2394
				continue;
2395
			s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
2396
			pr_debug("\t%s\n", sname);
L
Linus Torvalds 已提交
2397
		}
2398 2399
		switch (m) {
		case 0: /* executable */
2400 2401
			mod->core_layout.size = debug_align(mod->core_layout.size);
			mod->core_layout.text_size = mod->core_layout.size;
2402 2403
			break;
		case 1: /* RO: text and ro-data */
2404 2405
			mod->core_layout.size = debug_align(mod->core_layout.size);
			mod->core_layout.ro_size = mod->core_layout.size;
2406
			break;
J
Jessica Yu 已提交
2407 2408 2409 2410 2411
		case 2: /* RO after init */
			mod->core_layout.size = debug_align(mod->core_layout.size);
			mod->core_layout.ro_after_init_size = mod->core_layout.size;
			break;
		case 4: /* whole core */
2412
			mod->core_layout.size = debug_align(mod->core_layout.size);
2413 2414
			break;
		}
L
Linus Torvalds 已提交
2415 2416
	}

2417
	pr_debug("Init section allocation order:\n");
L
Linus Torvalds 已提交
2418
	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
2419 2420 2421
		for (i = 0; i < info->hdr->e_shnum; ++i) {
			Elf_Shdr *s = &info->sechdrs[i];
			const char *sname = info->secstrings + s->sh_name;
L
Linus Torvalds 已提交
2422 2423 2424 2425

			if ((s->sh_flags & masks[m][0]) != masks[m][0]
			    || (s->sh_flags & masks[m][1])
			    || s->sh_entsize != ~0UL
2426
			    || !strstarts(sname, ".init"))
L
Linus Torvalds 已提交
2427
				continue;
2428
			s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
L
Linus Torvalds 已提交
2429
					 | INIT_OFFSET_MASK);
2430
			pr_debug("\t%s\n", sname);
L
Linus Torvalds 已提交
2431
		}
2432 2433
		switch (m) {
		case 0: /* executable */
2434 2435
			mod->init_layout.size = debug_align(mod->init_layout.size);
			mod->init_layout.text_size = mod->init_layout.size;
2436 2437
			break;
		case 1: /* RO: text and ro-data */
2438 2439
			mod->init_layout.size = debug_align(mod->init_layout.size);
			mod->init_layout.ro_size = mod->init_layout.size;
2440
			break;
J
Jessica Yu 已提交
2441 2442 2443 2444 2445 2446 2447 2448
		case 2:
			/*
			 * RO after init doesn't apply to init_layout (only
			 * core_layout), so it just takes the value of ro_size.
			 */
			mod->init_layout.ro_after_init_size = mod->init_layout.ro_size;
			break;
		case 4: /* whole init */
2449
			mod->init_layout.size = debug_align(mod->init_layout.size);
2450 2451
			break;
		}
L
Linus Torvalds 已提交
2452 2453 2454 2455 2456 2457 2458 2459
	}
}

static void set_license(struct module *mod, const char *license)
{
	if (!license)
		license = "unspecified";

2460
	if (!license_is_gpl_compatible(license)) {
A
Andi Kleen 已提交
2461
		if (!test_taint(TAINT_PROPRIETARY_MODULE))
A
Andrew Morton 已提交
2462 2463
			pr_warn("%s: module license '%s' taints kernel.\n",
				mod->name, license);
2464 2465
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487
	}
}

/* Parse tag=value strings from .modinfo section */
static char *next_string(char *string, unsigned long *secsize)
{
	/* Skip non-zero chars */
	while (string[0]) {
		string++;
		if ((*secsize)-- <= 1)
			return NULL;
	}

	/* Skip any zero padding. */
	while (!string[0]) {
		string++;
		if ((*secsize)-- <= 1)
			return NULL;
	}
	return string;
}

2488
static char *get_modinfo(struct load_info *info, const char *tag)
L
Linus Torvalds 已提交
2489 2490 2491
{
	char *p;
	unsigned int taglen = strlen(tag);
2492 2493
	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
	unsigned long size = infosec->sh_size;
L
Linus Torvalds 已提交
2494

2495
	for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
L
Linus Torvalds 已提交
2496 2497 2498 2499 2500 2501
		if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
			return p + taglen + 1;
	}
	return NULL;
}

2502
static void setup_modinfo(struct module *mod, struct load_info *info)
2503 2504 2505 2506 2507 2508
{
	struct module_attribute *attr;
	int i;

	for (i = 0; (attr = modinfo_attrs[i]); i++) {
		if (attr->setup)
2509
			attr->setup(mod, get_modinfo(info, attr->attr.name));
2510 2511 2512
	}
}

2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
static void free_modinfo(struct module *mod)
{
	struct module_attribute *attr;
	int i;

	for (i = 0; (attr = modinfo_attrs[i]); i++) {
		if (attr->free)
			attr->free(mod);
	}
}

L
Linus Torvalds 已提交
2524
#ifdef CONFIG_KALLSYMS
2525 2526 2527 2528 2529 2530

/* lookup symbol in given range of kernel_symbols */
static const struct kernel_symbol *lookup_symbol(const char *name,
	const struct kernel_symbol *start,
	const struct kernel_symbol *stop)
{
2531 2532
	return bsearch(name, start, stop - start,
			sizeof(struct kernel_symbol), cmp_name);
2533 2534
}

2535 2536
static int is_exported(const char *name, unsigned long value,
		       const struct module *mod)
L
Linus Torvalds 已提交
2537
{
2538 2539 2540
	const struct kernel_symbol *ks;
	if (!mod)
		ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
2541
	else
2542 2543
		ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
	return ks != NULL && ks->value == value;
L
Linus Torvalds 已提交
2544 2545 2546
}

/* As per nm */
2547
static char elf_type(const Elf_Sym *sym, const struct load_info *info)
L
Linus Torvalds 已提交
2548
{
2549 2550
	const Elf_Shdr *sechdrs = info->sechdrs;

L
Linus Torvalds 已提交
2551 2552 2553 2554 2555 2556 2557 2558
	if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
		if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
			return 'v';
		else
			return 'w';
	}
	if (sym->st_shndx == SHN_UNDEF)
		return 'U';
2559
	if (sym->st_shndx == SHN_ABS || sym->st_shndx == info->index.pcpu)
L
Linus Torvalds 已提交
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579
		return 'a';
	if (sym->st_shndx >= SHN_LORESERVE)
		return '?';
	if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
		return 't';
	if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC
	    && sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
		if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
			return 'r';
		else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
			return 'g';
		else
			return 'd';
	}
	if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
		if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
			return 's';
		else
			return 'b';
	}
2580 2581
	if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
		      ".debug")) {
L
Linus Torvalds 已提交
2582
		return 'n';
2583
	}
L
Linus Torvalds 已提交
2584 2585 2586
	return '?';
}

2587
static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
2588
			unsigned int shnum, unsigned int pcpundx)
2589 2590 2591 2592 2593 2594 2595 2596
{
	const Elf_Shdr *sec;

	if (src->st_shndx == SHN_UNDEF
	    || src->st_shndx >= shnum
	    || !src->st_name)
		return false;

2597 2598 2599 2600 2601
#ifdef CONFIG_KALLSYMS_ALL
	if (src->st_shndx == pcpundx)
		return true;
#endif

2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
	sec = sechdrs + src->st_shndx;
	if (!(sec->sh_flags & SHF_ALLOC)
#ifndef CONFIG_KALLSYMS_ALL
	    || !(sec->sh_flags & SHF_EXECINSTR)
#endif
	    || (sec->sh_entsize & INIT_OFFSET_MASK))
		return false;

	return true;
}

2613 2614 2615 2616 2617 2618 2619
/*
 * We only allocate and copy the strings needed by the parts of symtab
 * we keep.  This is simple, but has the effect of making multiple
 * copies of duplicates.  We could be more sophisticated, see
 * linux-kernel thread starting with
 * <73defb5e4bca04a6431392cc341112b1@localhost>.
 */
2620
static void layout_symtab(struct module *mod, struct load_info *info)
2621
{
2622 2623
	Elf_Shdr *symsect = info->sechdrs + info->index.sym;
	Elf_Shdr *strsect = info->sechdrs + info->index.str;
2624
	const Elf_Sym *src;
2625
	unsigned int i, nsrc, ndst, strtab_size = 0;
2626 2627 2628

	/* Put symbol section at end of init part of module. */
	symsect->sh_flags |= SHF_ALLOC;
2629
	symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect,
2630
					 info->index.sym) | INIT_OFFSET_MASK;
2631
	pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
2632

2633
	src = (void *)info->hdr + symsect->sh_offset;
2634
	nsrc = symsect->sh_size / sizeof(*src);
2635

2636
	/* Compute total space required for the core symbols' strtab. */
2637
	for (ndst = i = 0; i < nsrc; i++) {
2638
		if (i == 0 || is_livepatch_module(mod) ||
2639 2640
		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
				   info->index.pcpu)) {
2641
			strtab_size += strlen(&info->strtab[src[i].st_name])+1;
2642
			ndst++;
2643
		}
2644
	}
2645 2646

	/* Append room for core symbols at end of core part. */
2647 2648 2649 2650
	info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
	info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
	mod->core_layout.size += strtab_size;
	mod->core_layout.size = debug_align(mod->core_layout.size);
2651

2652 2653
	/* Put string table section at end of init part of module. */
	strsect->sh_flags |= SHF_ALLOC;
2654
	strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect,
2655
					 info->index.str) | INIT_OFFSET_MASK;
2656
	pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
2657 2658 2659 2660 2661 2662 2663

	/* We'll tack temporary mod_kallsyms on the end. */
	mod->init_layout.size = ALIGN(mod->init_layout.size,
				      __alignof__(struct mod_kallsyms));
	info->mod_kallsyms_init_off = mod->init_layout.size;
	mod->init_layout.size += sizeof(struct mod_kallsyms);
	mod->init_layout.size = debug_align(mod->init_layout.size);
2664 2665
}

2666 2667 2668 2669 2670
/*
 * We use the full symtab and strtab which layout_symtab arranged to
 * be appended to the init section.  Later we switch to the cut-down
 * core-only ones.
 */
2671
static void add_kallsyms(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
2672
{
2673 2674 2675
	unsigned int i, ndst;
	const Elf_Sym *src;
	Elf_Sym *dst;
2676
	char *s;
2677
	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
L
Linus Torvalds 已提交
2678

2679 2680 2681 2682 2683
	/* Set up to point into init section. */
	mod->kallsyms = mod->init_layout.base + info->mod_kallsyms_init_off;

	mod->kallsyms->symtab = (void *)symsec->sh_addr;
	mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
2684
	/* Make sure we get permanent strtab: don't use info->strtab. */
2685
	mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
L
Linus Torvalds 已提交
2686 2687

	/* Set types up while we still have access to sections. */
2688 2689 2690 2691 2692 2693 2694 2695 2696
	for (i = 0; i < mod->kallsyms->num_symtab; i++)
		mod->kallsyms->symtab[i].st_info
			= elf_type(&mod->kallsyms->symtab[i], info);

	/* Now populate the cut down core kallsyms for after init. */
	mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs;
	mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
	src = mod->kallsyms->symtab;
	for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
2697
		if (i == 0 || is_livepatch_module(mod) ||
2698 2699
		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
				   info->index.pcpu)) {
2700
			dst[ndst] = src[i];
2701 2702
			dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
			s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
2703 2704
				     KSYM_NAME_LEN) + 1;
		}
2705
	}
2706
	mod->core_kallsyms.num_symtab = ndst;
L
Linus Torvalds 已提交
2707 2708
}
#else
2709
static inline void layout_symtab(struct module *mod, struct load_info *info)
2710 2711
{
}
2712

2713
static void add_kallsyms(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
2714 2715 2716 2717
{
}
#endif /* CONFIG_KALLSYMS */

2718
static void dynamic_debug_setup(struct module *mod, struct _ddebug *debug, unsigned int num)
2719
{
2720 2721
	if (!debug)
		return;
2722
#ifdef CONFIG_DYNAMIC_DEBUG
2723
	if (ddebug_add_module(debug, num, mod->name))
A
Andrew Morton 已提交
2724 2725
		pr_err("dynamic debug error adding module: %s\n",
			debug->modname);
2726
#endif
R
Rusty Russell 已提交
2727
}
2728

2729
static void dynamic_debug_remove(struct module *mod, struct _ddebug *debug)
2730 2731
{
	if (debug)
2732
		ddebug_remove_module(mod->name);
2733 2734
}

2735 2736
void * __weak module_alloc(unsigned long size)
{
R
Rusty Russell 已提交
2737
	return vmalloc_exec(size);
2738 2739
}

C
Catalin Marinas 已提交
2740
#ifdef CONFIG_DEBUG_KMEMLEAK
2741 2742
static void kmemleak_load_module(const struct module *mod,
				 const struct load_info *info)
C
Catalin Marinas 已提交
2743 2744 2745 2746
{
	unsigned int i;

	/* only scan the sections containing data */
2747
	kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
C
Catalin Marinas 已提交
2748

2749
	for (i = 1; i < info->hdr->e_shnum; i++) {
2750 2751 2752 2753
		/* Scan all writable sections that's not executable */
		if (!(info->sechdrs[i].sh_flags & SHF_ALLOC) ||
		    !(info->sechdrs[i].sh_flags & SHF_WRITE) ||
		    (info->sechdrs[i].sh_flags & SHF_EXECINSTR))
C
Catalin Marinas 已提交
2754 2755
			continue;

2756 2757
		kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
				   info->sechdrs[i].sh_size, GFP_KERNEL);
C
Catalin Marinas 已提交
2758 2759 2760
	}
}
#else
2761 2762
static inline void kmemleak_load_module(const struct module *mod,
					const struct load_info *info)
C
Catalin Marinas 已提交
2763 2764 2765 2766
{
}
#endif

R
Rusty Russell 已提交
2767
#ifdef CONFIG_MODULE_SIG
2768
static int module_sig_check(struct load_info *info, int flags)
R
Rusty Russell 已提交
2769 2770
{
	int err = -ENOKEY;
2771 2772
	const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
	const void *mod = info->hdr;
2773

2774 2775 2776 2777 2778 2779
	/*
	 * Require flags == 0, as a module with version information
	 * removed is no longer the module that was signed
	 */
	if (flags == 0 &&
	    info->len > markerlen &&
2780
	    memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
2781
		/* We truncate the module to discard the signature */
2782 2783
		info->len -= markerlen;
		err = mod_verify_sig(mod, &info->len);
R
Rusty Russell 已提交
2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797
	}

	if (!err) {
		info->sig_ok = true;
		return 0;
	}

	/* Not having a signature is only an error if we're strict. */
	if (err == -ENOKEY && !sig_enforce)
		err = 0;

	return err;
}
#else /* !CONFIG_MODULE_SIG */
2798
static int module_sig_check(struct load_info *info, int flags)
R
Rusty Russell 已提交
2799 2800 2801 2802 2803
{
	return 0;
}
#endif /* !CONFIG_MODULE_SIG */

2804 2805
/* Sanity checks against invalid binaries, wrong arch, weird elf version. */
static int elf_header_check(struct load_info *info)
2806
{
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819
	if (info->len < sizeof(*(info->hdr)))
		return -ENOEXEC;

	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
	    || info->hdr->e_type != ET_REL
	    || !elf_check_arch(info->hdr)
	    || info->hdr->e_shentsize != sizeof(Elf_Shdr))
		return -ENOEXEC;

	if (info->hdr->e_shoff >= info->len
	    || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
		info->len - info->hdr->e_shoff))
		return -ENOEXEC;
2820

2821 2822 2823
	return 0;
}

2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840
#define COPY_CHUNK_SIZE (16*PAGE_SIZE)

static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
{
	do {
		unsigned long n = min(len, COPY_CHUNK_SIZE);

		if (copy_from_user(dst, usrc, n) != 0)
			return -EFAULT;
		cond_resched();
		dst += n;
		usrc += n;
		len -= n;
	} while (len);
	return 0;
}

2841
#ifdef CONFIG_LIVEPATCH
2842
static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
2843
{
2844 2845 2846
	if (get_modinfo(info, "livepatch")) {
		mod->klp = true;
		add_taint_module(mod, TAINT_LIVEPATCH, LOCKDEP_STILL_OK);
2847 2848
		pr_notice_once("%s: tainting kernel with TAINT_LIVEPATCH\n",
			       mod->name);
2849
	}
2850 2851 2852 2853

	return 0;
}
#else /* !CONFIG_LIVEPATCH */
2854
static int check_modinfo_livepatch(struct module *mod, struct load_info *info)
2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865
{
	if (get_modinfo(info, "livepatch")) {
		pr_err("%s: module is marked as livepatch module, but livepatch support is disabled",
		       mod->name);
		return -ENOEXEC;
	}

	return 0;
}
#endif /* CONFIG_LIVEPATCH */

2866 2867 2868
/* Sets info->hdr and info->len. */
static int copy_module_from_user(const void __user *umod, unsigned long len,
				  struct load_info *info)
2869 2870 2871
{
	int err;

2872 2873
	info->len = len;
	if (info->len < sizeof(*(info->hdr)))
2874 2875
		return -ENOEXEC;

2876
	err = security_kernel_read_file(NULL, READING_MODULE);
2877 2878 2879
	if (err)
		return err;

2880
	/* Suck in entire file: we'll want most of it. */
2881
	info->hdr = __vmalloc(info->len,
2882
			GFP_KERNEL | __GFP_NOWARN, PAGE_KERNEL);
2883
	if (!info->hdr)
2884 2885
		return -ENOMEM;

2886
	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
2887 2888
		vfree(info->hdr);
		return -EFAULT;
2889 2890
	}

2891 2892 2893
	return 0;
}

R
Rusty Russell 已提交
2894 2895 2896 2897 2898
static void free_copy(struct load_info *info)
{
	vfree(info->hdr);
}

2899
static int rewrite_section_headers(struct load_info *info, int flags)
2900 2901 2902 2903 2904 2905 2906 2907 2908 2909
{
	unsigned int i;

	/* This should always be true, but let's be sure. */
	info->sechdrs[0].sh_addr = 0;

	for (i = 1; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *shdr = &info->sechdrs[i];
		if (shdr->sh_type != SHT_NOBITS
		    && info->len < shdr->sh_offset + shdr->sh_size) {
A
Andrew Morton 已提交
2910
			pr_err("Module len %lu truncated\n", info->len);
2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
			return -ENOEXEC;
		}

		/* Mark all sections sh_addr with their address in the
		   temporary image. */
		shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;

#ifndef CONFIG_MODULE_UNLOAD
		/* Don't load .exit sections */
		if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
			shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
#endif
	}
2924 2925

	/* Track but don't keep modinfo and version sections. */
2926 2927 2928 2929
	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
		info->index.vers = 0; /* Pretend no __versions section! */
	else
		info->index.vers = find_sec(info, "__versions");
K
Kees Cook 已提交
2930 2931
	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;

2932
	info->index.info = find_sec(info, ".modinfo");
K
Kees Cook 已提交
2933 2934 2935 2936
	if (!info->index.info)
		info->name = "(missing .modinfo section)";
	else
		info->name = get_modinfo(info, "name");
2937
	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
K
Kees Cook 已提交
2938

2939 2940 2941
	return 0;
}

L
Linus Torvalds 已提交
2942 2943 2944 2945 2946 2947 2948 2949
/*
 * Set up our basic convenience variables (pointers to section headers,
 * search for module section index etc), and do some basic section
 * verification.
 *
 * Return the temporary module pointer (we'll replace it with the final
 * one when we move the module sections around).
 */
2950
static struct module *setup_load_info(struct load_info *info, int flags)
L
Linus Torvalds 已提交
2951 2952
{
	unsigned int i;
2953
	int err;
L
Linus Torvalds 已提交
2954 2955 2956 2957
	struct module *mod;

	/* Set up the convenience variables */
	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
2958 2959
	info->secstrings = (void *)info->hdr
		+ info->sechdrs[info->hdr->e_shstrndx].sh_offset;
L
Linus Torvalds 已提交
2960

2961
	err = rewrite_section_headers(info, flags);
2962 2963
	if (err)
		return ERR_PTR(err);
L
Linus Torvalds 已提交
2964

2965 2966
	/* Find internal symbols and strings. */
	for (i = 1; i < info->hdr->e_shnum; i++) {
L
Linus Torvalds 已提交
2967 2968 2969
		if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
			info->index.sym = i;
			info->index.str = info->sechdrs[i].sh_link;
2970 2971 2972
			info->strtab = (char *)info->hdr
				+ info->sechdrs[info->index.str].sh_offset;
			break;
L
Linus Torvalds 已提交
2973 2974 2975
		}
	}

2976
	info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
L
Linus Torvalds 已提交
2977
	if (!info->index.mod) {
K
Kees Cook 已提交
2978 2979
		pr_warn("%s: No module found in object\n",
			info->name ?: "(missing .modinfo name field)");
L
Linus Torvalds 已提交
2980 2981 2982 2983 2984
		return ERR_PTR(-ENOEXEC);
	}
	/* This is temporary: point mod into copy of data. */
	mod = (void *)info->sechdrs[info->index.mod].sh_addr;

K
Kees Cook 已提交
2985 2986 2987 2988 2989 2990 2991
	/*
	 * If we didn't load the .modinfo 'name' field, fall back to
	 * on-disk struct mod 'name' field.
	 */
	if (!info->name)
		info->name = mod->name;

L
Linus Torvalds 已提交
2992
	if (info->index.sym == 0) {
K
Kees Cook 已提交
2993
		pr_warn("%s: module has no symbols (stripped?)\n", info->name);
L
Linus Torvalds 已提交
2994 2995 2996
		return ERR_PTR(-ENOEXEC);
	}

2997
	info->index.pcpu = find_pcpusec(info);
L
Linus Torvalds 已提交
2998 2999

	/* Check module struct version now, before we try to use module. */
3000
	if (!check_modstruct_version(info, mod))
L
Linus Torvalds 已提交
3001 3002 3003 3004 3005
		return ERR_PTR(-ENOEXEC);

	return mod;
}

3006
static int check_modinfo(struct module *mod, struct load_info *info, int flags)
3007
{
3008
	const char *modmagic = get_modinfo(info, "vermagic");
3009 3010
	int err;

3011 3012 3013
	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
		modmagic = NULL;

3014 3015 3016 3017 3018
	/* This is allowed: modprobe --force will invalidate it. */
	if (!modmagic) {
		err = try_to_force_load(mod, "bad vermagic");
		if (err)
			return err;
3019
	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
A
Andrew Morton 已提交
3020
		pr_err("%s: version magic '%s' should be '%s'\n",
K
Kees Cook 已提交
3021
		       info->name, modmagic, vermagic);
3022 3023 3024
		return -ENOEXEC;
	}

3025 3026 3027 3028
	if (!get_modinfo(info, "intree")) {
		if (!test_taint(TAINT_OOT_MODULE))
			pr_warn("%s: loading out-of-tree module taints kernel.\n",
				mod->name);
3029
		add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
3030
	}
3031

3032
	if (get_modinfo(info, "staging")) {
3033
		add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
A
Andrew Morton 已提交
3034 3035
		pr_warn("%s: module is from the staging directory, the quality "
			"is unknown, you have been warned.\n", mod->name);
3036
	}
3037

3038
	err = check_modinfo_livepatch(mod, info);
3039 3040 3041
	if (err)
		return err;

3042
	/* Set up license info based on the info section */
3043
	set_license(mod, get_modinfo(info, "license"));
3044

3045 3046 3047
	return 0;
}

3048
static int find_module_sections(struct module *mod, struct load_info *info)
L
Linus Torvalds 已提交
3049
{
3050
	mod->kp = section_objs(info, "__param",
L
Linus Torvalds 已提交
3051
			       sizeof(*mod->kp), &mod->num_kp);
3052
	mod->syms = section_objs(info, "__ksymtab",
L
Linus Torvalds 已提交
3053
				 sizeof(*mod->syms), &mod->num_syms);
3054 3055
	mod->crcs = section_addr(info, "__kcrctab");
	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
L
Linus Torvalds 已提交
3056 3057
				     sizeof(*mod->gpl_syms),
				     &mod->num_gpl_syms);
3058 3059
	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
	mod->gpl_future_syms = section_objs(info,
L
Linus Torvalds 已提交
3060 3061 3062
					    "__ksymtab_gpl_future",
					    sizeof(*mod->gpl_future_syms),
					    &mod->num_gpl_future_syms);
3063
	mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
L
Linus Torvalds 已提交
3064 3065

#ifdef CONFIG_UNUSED_SYMBOLS
3066
	mod->unused_syms = section_objs(info, "__ksymtab_unused",
L
Linus Torvalds 已提交
3067 3068
					sizeof(*mod->unused_syms),
					&mod->num_unused_syms);
3069 3070
	mod->unused_crcs = section_addr(info, "__kcrctab_unused");
	mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
L
Linus Torvalds 已提交
3071 3072
					    sizeof(*mod->unused_gpl_syms),
					    &mod->num_unused_gpl_syms);
3073
	mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
L
Linus Torvalds 已提交
3074 3075
#endif
#ifdef CONFIG_CONSTRUCTORS
3076
	mod->ctors = section_objs(info, ".ctors",
L
Linus Torvalds 已提交
3077
				  sizeof(*mod->ctors), &mod->num_ctors);
3078 3079 3080 3081 3082 3083 3084 3085
	if (!mod->ctors)
		mod->ctors = section_objs(info, ".init_array",
				sizeof(*mod->ctors), &mod->num_ctors);
	else if (find_sec(info, ".init_array")) {
		/*
		 * This shouldn't happen with same compiler and binutils
		 * building all parts of the module.
		 */
3086
		pr_warn("%s: has both .ctors and .init_array.\n",
3087 3088 3089
		       mod->name);
		return -EINVAL;
	}
L
Linus Torvalds 已提交
3090 3091 3092
#endif

#ifdef CONFIG_TRACEPOINTS
3093 3094 3095
	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
					     sizeof(*mod->tracepoints_ptrs),
					     &mod->num_tracepoints);
L
Linus Torvalds 已提交
3096
#endif
3097 3098 3099 3100 3101
#ifdef HAVE_JUMP_LABEL
	mod->jump_entries = section_objs(info, "__jump_table",
					sizeof(*mod->jump_entries),
					&mod->num_jump_entries);
#endif
L
Linus Torvalds 已提交
3102
#ifdef CONFIG_EVENT_TRACING
3103
	mod->trace_events = section_objs(info, "_ftrace_events",
L
Linus Torvalds 已提交
3104 3105
					 sizeof(*mod->trace_events),
					 &mod->num_trace_events);
3106 3107 3108
	mod->trace_evals = section_objs(info, "_ftrace_eval_map",
					sizeof(*mod->trace_evals),
					&mod->num_trace_evals);
L
Linus Torvalds 已提交
3109
#endif
3110 3111 3112 3113 3114
#ifdef CONFIG_TRACING
	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
					 sizeof(*mod->trace_bprintk_fmt_start),
					 &mod->num_trace_bprintk_fmt);
#endif
L
Linus Torvalds 已提交
3115 3116
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
	/* sechdrs[0].sh_size is always zero */
3117
	mod->ftrace_callsites = section_objs(info, "__mcount_loc",
L
Linus Torvalds 已提交
3118 3119 3120
					     sizeof(*mod->ftrace_callsites),
					     &mod->num_ftrace_callsites);
#endif
3121 3122 3123 3124 3125
#ifdef CONFIG_BPF_KPROBE_OVERRIDE
	mod->kprobe_ei_funcs = section_objs(info, "_kprobe_error_inject_list",
					    sizeof(*mod->kprobe_ei_funcs),
					    &mod->num_kprobe_ei_funcs);
#endif
3126 3127 3128
	mod->extable = section_objs(info, "__ex_table",
				    sizeof(*mod->extable), &mod->num_exentries);

3129
	if (section_addr(info, "__obsparm"))
A
Andrew Morton 已提交
3130
		pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
3131 3132 3133

	info->debug = section_objs(info, "__verbose",
				   sizeof(*info->debug), &info->num_debug);
3134 3135

	return 0;
L
Linus Torvalds 已提交
3136 3137
}

3138
static int move_module(struct module *mod, struct load_info *info)
3139 3140 3141 3142 3143
{
	int i;
	void *ptr;

	/* Do the allocs. */
3144
	ptr = module_alloc(mod->core_layout.size);
3145 3146 3147 3148 3149 3150 3151
	/*
	 * The pointer to this block is stored in the module structure
	 * which is inside the block. Just mark it as not being a
	 * leak.
	 */
	kmemleak_not_leak(ptr);
	if (!ptr)
R
Rusty Russell 已提交
3152
		return -ENOMEM;
3153

3154 3155
	memset(ptr, 0, mod->core_layout.size);
	mod->core_layout.base = ptr;
3156

3157 3158
	if (mod->init_layout.size) {
		ptr = module_alloc(mod->init_layout.size);
R
Rusty Russell 已提交
3159 3160 3161 3162 3163 3164 3165 3166
		/*
		 * The pointer to this block is stored in the module structure
		 * which is inside the block. This block doesn't need to be
		 * scanned as it contains data and code that will be freed
		 * after the module is initialized.
		 */
		kmemleak_ignore(ptr);
		if (!ptr) {
3167
			module_memfree(mod->core_layout.base);
R
Rusty Russell 已提交
3168 3169
			return -ENOMEM;
		}
3170 3171
		memset(ptr, 0, mod->init_layout.size);
		mod->init_layout.base = ptr;
R
Rusty Russell 已提交
3172
	} else
3173
		mod->init_layout.base = NULL;
3174 3175

	/* Transfer each section which specifies SHF_ALLOC */
3176
	pr_debug("final section addresses:\n");
3177
	for (i = 0; i < info->hdr->e_shnum; i++) {
3178
		void *dest;
3179
		Elf_Shdr *shdr = &info->sechdrs[i];
3180

3181
		if (!(shdr->sh_flags & SHF_ALLOC))
3182 3183
			continue;

3184
		if (shdr->sh_entsize & INIT_OFFSET_MASK)
3185
			dest = mod->init_layout.base
3186
				+ (shdr->sh_entsize & ~INIT_OFFSET_MASK);
3187
		else
3188
			dest = mod->core_layout.base + shdr->sh_entsize;
3189

3190 3191
		if (shdr->sh_type != SHT_NOBITS)
			memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
3192
		/* Update sh_addr to point to copy in image. */
3193
		shdr->sh_addr = (unsigned long)dest;
3194 3195
		pr_debug("\t0x%lx %s\n",
			 (long)shdr->sh_addr, info->secstrings + shdr->sh_name);
3196
	}
R
Rusty Russell 已提交
3197 3198

	return 0;
3199 3200
}

3201
static int check_module_license_and_versions(struct module *mod)
3202
{
3203 3204
	int prev_taint = test_taint(TAINT_PROPRIETARY_MODULE);

3205 3206 3207 3208 3209 3210
	/*
	 * ndiswrapper is under GPL by itself, but loads proprietary modules.
	 * Don't use add_taint_module(), as it would prevent ndiswrapper from
	 * using GPL-only symbols it needs.
	 */
	if (strcmp(mod->name, "ndiswrapper") == 0)
3211
		add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
3212 3213 3214

	/* driverloader was caught wrongly pretending to be under GPL */
	if (strcmp(mod->name, "driverloader") == 0)
3215 3216
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
3217

3218 3219
	/* lve claims to be GPL but upstream won't provide source */
	if (strcmp(mod->name, "lve") == 0)
3220 3221
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
3222

3223 3224 3225
	if (!prev_taint && test_taint(TAINT_PROPRIETARY_MODULE))
		pr_warn("%s: module license taints kernel.\n", mod->name);

3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254
#ifdef CONFIG_MODVERSIONS
	if ((mod->num_syms && !mod->crcs)
	    || (mod->num_gpl_syms && !mod->gpl_crcs)
	    || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
#ifdef CONFIG_UNUSED_SYMBOLS
	    || (mod->num_unused_syms && !mod->unused_crcs)
	    || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
#endif
		) {
		return try_to_force_load(mod,
					 "no versions for exported symbols");
	}
#endif
	return 0;
}

static void flush_module_icache(const struct module *mod)
{
	mm_segment_t old_fs;

	/* flush the icache in correct context */
	old_fs = get_fs();
	set_fs(KERNEL_DS);

	/*
	 * Flush the instruction cache, since we've played with text.
	 * Do it before processing of module parameters, so the module
	 * can provide parameter accessor functions of its own.
	 */
3255 3256 3257 3258 3259 3260
	if (mod->init_layout.base)
		flush_icache_range((unsigned long)mod->init_layout.base,
				   (unsigned long)mod->init_layout.base
				   + mod->init_layout.size);
	flush_icache_range((unsigned long)mod->core_layout.base,
			   (unsigned long)mod->core_layout.base + mod->core_layout.size);
3261 3262 3263 3264

	set_fs(old_fs);
}

3265 3266 3267 3268 3269 3270 3271 3272
int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
				     Elf_Shdr *sechdrs,
				     char *secstrings,
				     struct module *mod)
{
	return 0;
}

3273 3274
/* module_blacklist is a comma-separated list of module names */
static char *module_blacklist;
3275
static bool blacklisted(const char *module_name)
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293
{
	const char *p;
	size_t len;

	if (!module_blacklist)
		return false;

	for (p = module_blacklist; *p; p += len) {
		len = strcspn(p, ",");
		if (strlen(module_name) == len && !memcmp(module_name, p, len))
			return true;
		if (p[len] == ',')
			len++;
	}
	return false;
}
core_param(module_blacklist, module_blacklist, charp, 0400);

3294
static struct module *layout_and_allocate(struct load_info *info, int flags)
L
Linus Torvalds 已提交
3295
{
R
Rusty Russell 已提交
3296
	/* Module within temporary copy. */
L
Linus Torvalds 已提交
3297
	struct module *mod;
J
Jessica Yu 已提交
3298
	unsigned int ndx;
R
Rusty Russell 已提交
3299
	int err;
3300

3301
	mod = setup_load_info(info, flags);
R
Rusty Russell 已提交
3302 3303
	if (IS_ERR(mod))
		return mod;
L
Linus Torvalds 已提交
3304

K
Kees Cook 已提交
3305
	if (blacklisted(info->name))
3306 3307
		return ERR_PTR(-EPERM);

3308
	err = check_modinfo(mod, info, flags);
3309 3310
	if (err)
		return ERR_PTR(err);
L
Linus Torvalds 已提交
3311 3312

	/* Allow arches to frob section contents and sizes.  */
3313 3314
	err = module_frob_arch_sections(info->hdr, info->sechdrs,
					info->secstrings, mod);
L
Linus Torvalds 已提交
3315
	if (err < 0)
3316
		return ERR_PTR(err);
L
Linus Torvalds 已提交
3317

3318 3319
	/* We will do a special allocation for per-cpu sections later. */
	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
L
Linus Torvalds 已提交
3320

J
Jessica Yu 已提交
3321 3322 3323 3324 3325 3326 3327 3328 3329
	/*
	 * Mark ro_after_init section with SHF_RO_AFTER_INIT so that
	 * layout_sections() can put it in the right place.
	 * Note: ro_after_init sections also have SHF_{WRITE,ALLOC} set.
	 */
	ndx = find_sec(info, ".data..ro_after_init");
	if (ndx)
		info->sechdrs[ndx].sh_flags |= SHF_RO_AFTER_INIT;

L
Linus Torvalds 已提交
3330 3331 3332
	/* Determine total sizes, and put offsets in sh_entsize.  For now
	   this is done generically; there doesn't appear to be any
	   special cases for the architectures. */
3333 3334
	layout_sections(mod, info);
	layout_symtab(mod, info);
L
Linus Torvalds 已提交
3335

3336
	/* Allocate and move to the final place */
3337
	err = move_module(mod, info);
R
Rusty Russell 已提交
3338
	if (err)
3339
		return ERR_PTR(err);
R
Rusty Russell 已提交
3340 3341 3342

	/* Module has been copied to its final place now: return it. */
	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
3343
	kmemleak_load_module(mod, info);
R
Rusty Russell 已提交
3344 3345 3346 3347 3348 3349 3350
	return mod;
}

/* mod is no longer valid after this! */
static void module_deallocate(struct module *mod, struct load_info *info)
{
	percpu_modfree(mod);
3351
	module_arch_freeing_init(mod);
3352 3353
	module_memfree(mod->init_layout.base);
	module_memfree(mod->core_layout.base);
R
Rusty Russell 已提交
3354 3355
}

3356 3357 3358 3359 3360 3361 3362
int __weak module_finalize(const Elf_Ehdr *hdr,
			   const Elf_Shdr *sechdrs,
			   struct module *me)
{
	return 0;
}

3363 3364
static int post_relocation(struct module *mod, const struct load_info *info)
{
3365
	/* Sort exception table now relocations are done. */
3366 3367 3368 3369 3370 3371
	sort_extable(mod->extable, mod->extable + mod->num_exentries);

	/* Copy relocated percpu area over. */
	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
		       info->sechdrs[info->index.pcpu].sh_size);

3372
	/* Setup kallsyms-specific fields. */
3373 3374 3375 3376 3377 3378
	add_kallsyms(mod, info);

	/* Arch-specific module finalizing. */
	return module_finalize(info->hdr, info->sechdrs, mod);
}

3379 3380 3381 3382 3383 3384
/* Is this module of this name done loading?  No locks held. */
static bool finished_loading(const char *name)
{
	struct module *mod;
	bool ret;

3385 3386 3387 3388 3389 3390
	/*
	 * The module_mutex should not be a heavily contended lock;
	 * if we get the occasional sleep here, we'll go an extra iteration
	 * in the wait_event_interruptible(), which is harmless.
	 */
	sched_annotate_sleep();
3391
	mutex_lock(&module_mutex);
3392
	mod = find_module_all(name, strlen(name), true);
3393 3394
	ret = !mod || mod->state == MODULE_STATE_LIVE
		|| mod->state == MODULE_STATE_GOING;
3395 3396 3397 3398 3399
	mutex_unlock(&module_mutex);

	return ret;
}

3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410
/* Call module constructors. */
static void do_mod_ctors(struct module *mod)
{
#ifdef CONFIG_CONSTRUCTORS
	unsigned long i;

	for (i = 0; i < mod->num_ctors; i++)
		mod->ctors[i]();
#endif
}

3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423
/* For freeing module_init on success, in case kallsyms traversing */
struct mod_initfree {
	struct rcu_head rcu;
	void *module_init;
};

static void do_free_init(struct rcu_head *head)
{
	struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
	module_memfree(m->module_init);
	kfree(m);
}

3424 3425 3426 3427 3428 3429 3430
/*
 * This is where the real work happens.
 *
 * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
 * helper command 'lx-symbols'.
 */
static noinline int do_init_module(struct module *mod)
3431 3432
{
	int ret = 0;
3433 3434 3435 3436 3437 3438 3439
	struct mod_initfree *freeinit;

	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
	if (!freeinit) {
		ret = -ENOMEM;
		goto fail;
	}
3440
	freeinit->module_init = mod->init_layout.base;
3441

3442 3443 3444 3445 3446 3447
	/*
	 * We want to find out whether @mod uses async during init.  Clear
	 * PF_USED_ASYNC.  async_schedule*() will set it.
	 */
	current->flags &= ~PF_USED_ASYNC;

3448 3449 3450 3451 3452
	do_mod_ctors(mod);
	/* Start the module */
	if (mod->init != NULL)
		ret = do_one_initcall(mod->init);
	if (ret < 0) {
3453
		goto fail_free_freeinit;
3454 3455
	}
	if (ret > 0) {
A
Andrew Morton 已提交
3456 3457 3458 3459
		pr_warn("%s: '%s'->init suspiciously returned %d, it should "
			"follow 0/-E convention\n"
			"%s: loading module anyway...\n",
			__func__, mod->name, ret, __func__);
3460 3461 3462 3463 3464 3465 3466 3467
		dump_stack();
	}

	/* Now it's a first class citizen! */
	mod->state = MODULE_STATE_LIVE;
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_LIVE, mod);

3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484
	/*
	 * We need to finish all async code before the module init sequence
	 * is done.  This has potential to deadlock.  For example, a newly
	 * detected block device can trigger request_module() of the
	 * default iosched from async probing task.  Once userland helper
	 * reaches here, async_synchronize_full() will wait on the async
	 * task waiting on request_module() and deadlock.
	 *
	 * This deadlock is avoided by perfomring async_synchronize_full()
	 * iff module init queued any async jobs.  This isn't a full
	 * solution as it will deadlock the same if module loading from
	 * async jobs nests more than once; however, due to the various
	 * constraints, this hack seems to be the best option for now.
	 * Please refer to the following thread for details.
	 *
	 * http://thread.gmane.org/gmane.linux.kernel/1420814
	 */
3485
	if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC))
3486
		async_synchronize_full();
3487

3488
	ftrace_free_mem(mod, mod->init_layout.base, mod->init_layout.base +
3489
			mod->init_layout.size);
3490 3491 3492 3493 3494
	mutex_lock(&module_mutex);
	/* Drop initial reference. */
	module_put(mod);
	trim_init_extable(mod);
#ifdef CONFIG_KALLSYMS
3495 3496
	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
3497
#endif
J
Jessica Yu 已提交
3498
	module_enable_ro(mod, true);
3499
	mod_tree_remove_init(mod);
R
Rusty Russell 已提交
3500
	disable_ro_nx(&mod->init_layout);
3501
	module_arch_freeing_init(mod);
3502 3503 3504
	mod->init_layout.base = NULL;
	mod->init_layout.size = 0;
	mod->init_layout.ro_size = 0;
J
Jessica Yu 已提交
3505
	mod->init_layout.ro_after_init_size = 0;
3506
	mod->init_layout.text_size = 0;
3507 3508
	/*
	 * We want to free module_init, but be aware that kallsyms may be
3509 3510 3511
	 * walking this with preempt disabled.  In all the failure paths, we
	 * call synchronize_sched(), but we don't want to slow down the success
	 * path, so use actual RCU here.
3512
	 */
3513
	call_rcu_sched(&freeinit->rcu, do_free_init);
3514 3515 3516 3517
	mutex_unlock(&module_mutex);
	wake_up_all(&module_wq);

	return 0;
3518 3519 3520 3521 3522 3523 3524 3525 3526 3527

fail_free_freeinit:
	kfree(freeinit);
fail:
	/* Try to protect us from buggy refcounters. */
	mod->state = MODULE_STATE_GOING;
	synchronize_sched();
	module_put(mod);
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
3528
	klp_module_going(mod);
3529
	ftrace_release_mod(mod);
3530 3531 3532
	free_module(mod);
	wake_up_all(&module_wq);
	return ret;
3533 3534 3535 3536 3537 3538 3539 3540 3541 3542
}

static int may_init_module(void)
{
	if (!capable(CAP_SYS_MODULE) || modules_disabled)
		return -EPERM;

	return 0;
}

3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556
/*
 * We try to place it in the list now to make sure it's unique before
 * we dedicate too many resources.  In particular, temporary percpu
 * memory exhaustion.
 */
static int add_unformed_module(struct module *mod)
{
	int err;
	struct module *old;

	mod->state = MODULE_STATE_UNFORMED;

again:
	mutex_lock(&module_mutex);
3557 3558
	old = find_module_all(mod->name, strlen(mod->name), true);
	if (old != NULL) {
3559 3560 3561 3562
		if (old->state == MODULE_STATE_COMING
		    || old->state == MODULE_STATE_UNFORMED) {
			/* Wait in case it fails to load. */
			mutex_unlock(&module_mutex);
3563 3564
			err = wait_event_interruptible(module_wq,
					       finished_loading(mod->name));
3565 3566 3567 3568 3569 3570 3571
			if (err)
				goto out_unlocked;
			goto again;
		}
		err = -EEXIST;
		goto out;
	}
3572
	mod_update_bounds(mod);
3573
	list_add_rcu(&mod->list, &modules);
3574
	mod_tree_insert(mod);
3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596
	err = 0;

out:
	mutex_unlock(&module_mutex);
out_unlocked:
	return err;
}

static int complete_formation(struct module *mod, struct load_info *info)
{
	int err;

	mutex_lock(&module_mutex);

	/* Find duplicate symbols (must be called under lock). */
	err = verify_export_symbols(mod);
	if (err < 0)
		goto out;

	/* This relies on module_mutex for list integrity. */
	module_bug_finalize(info->hdr, info->sechdrs, mod);

J
Jessica Yu 已提交
3597
	module_enable_ro(mod, false);
R
Rusty Russell 已提交
3598
	module_enable_nx(mod);
3599

3600 3601 3602
	/* Mark state as coming so strong_try_module_get() ignores us,
	 * but kallsyms etc. can see us. */
	mod->state = MODULE_STATE_COMING;
3603 3604 3605
	mutex_unlock(&module_mutex);

	return 0;
3606 3607 3608 3609 3610 3611

out:
	mutex_unlock(&module_mutex);
	return err;
}

3612 3613
static int prepare_coming_module(struct module *mod)
{
3614 3615
	int err;

3616
	ftrace_module_enable(mod);
3617 3618 3619 3620
	err = klp_module_coming(mod);
	if (err)
		return err;

3621 3622 3623 3624 3625
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_COMING, mod);
	return 0;
}

3626 3627
static int unknown_module_param_cb(char *param, char *val, const char *modname,
				   void *arg)
3628
{
3629 3630 3631 3632 3633 3634 3635 3636
	struct module *mod = arg;
	int ret;

	if (strcmp(param, "async_probe") == 0) {
		mod->async_probe_requested = true;
		return 0;
	}

3637
	/* Check for magic 'dyndbg' arg */
3638
	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
A
Andrew Morton 已提交
3639 3640
	if (ret != 0)
		pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
3641 3642 3643
	return 0;
}

R
Rusty Russell 已提交
3644 3645
/* Allocate and load the module: note that size of section 0 is always
   zero, and we rely on this for optional sections. */
3646 3647
static int load_module(struct load_info *info, const char __user *uargs,
		       int flags)
R
Rusty Russell 已提交
3648
{
3649
	struct module *mod;
R
Rusty Russell 已提交
3650
	long err;
3651
	char *after_dashes;
R
Rusty Russell 已提交
3652

3653
	err = module_sig_check(info, flags);
3654 3655
	if (err)
		goto free_copy;
R
Rusty Russell 已提交
3656

3657
	err = elf_header_check(info);
R
Rusty Russell 已提交
3658
	if (err)
3659
		goto free_copy;
R
Rusty Russell 已提交
3660 3661

	/* Figure out module layout, and allocate all the memory. */
3662
	mod = layout_and_allocate(info, flags);
3663 3664
	if (IS_ERR(mod)) {
		err = PTR_ERR(mod);
R
Rusty Russell 已提交
3665
		goto free_copy;
L
Linus Torvalds 已提交
3666 3667
	}

3668 3669
	audit_log_kern_module(mod->name);

3670 3671 3672
	/* Reserve our place in the list. */
	err = add_unformed_module(mod);
	if (err)
3673 3674
		goto free_module;

R
Rusty Russell 已提交
3675
#ifdef CONFIG_MODULE_SIG
3676
	mod->sig_ok = info->sig_ok;
3677
	if (!mod->sig_ok) {
A
Andrew Morton 已提交
3678
		pr_notice_once("%s: module verification failed: signature "
3679
			       "and/or required key missing - tainting "
A
Andrew Morton 已提交
3680
			       "kernel\n", mod->name);
3681
		add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
3682
	}
R
Rusty Russell 已提交
3683 3684
#endif

3685
	/* To avoid stressing percpu allocator, do this once we're unique. */
R
Rusty Russell 已提交
3686
	err = percpu_modalloc(mod, info);
3687 3688 3689
	if (err)
		goto unlink_mod;

3690
	/* Now module is in final location, initialize linked lists, etc. */
3691 3692
	err = module_unload_init(mod);
	if (err)
3693
		goto unlink_mod;
L
Linus Torvalds 已提交
3694

3695
	init_param_lock(mod);
3696

3697 3698
	/* Now we've got everything in the final locations, we can
	 * find optional sections. */
3699 3700 3701
	err = find_module_sections(mod, info);
	if (err)
		goto free_unload;
3702

3703
	err = check_module_license_and_versions(mod);
3704 3705
	if (err)
		goto free_unload;
3706

3707
	/* Set up MODINFO_ATTR fields */
3708
	setup_modinfo(mod, info);
3709

L
Linus Torvalds 已提交
3710
	/* Fix up syms, so that st_value is a pointer to location. */
3711
	err = simplify_symbols(mod, info);
L
Linus Torvalds 已提交
3712
	if (err < 0)
R
Rusty Russell 已提交
3713
		goto free_modinfo;
L
Linus Torvalds 已提交
3714

3715
	err = apply_relocations(mod, info);
3716
	if (err < 0)
R
Rusty Russell 已提交
3717
		goto free_modinfo;
L
Linus Torvalds 已提交
3718

3719
	err = post_relocation(mod, info);
L
Linus Torvalds 已提交
3720
	if (err < 0)
R
Rusty Russell 已提交
3721
		goto free_modinfo;
L
Linus Torvalds 已提交
3722

3723
	flush_module_icache(mod);
3724

3725 3726 3727 3728 3729 3730
	/* Now copy in args */
	mod->args = strndup_user(uargs, ~0UL >> 1);
	if (IS_ERR(mod->args)) {
		err = PTR_ERR(mod->args);
		goto free_arch_cleanup;
	}
R
Rusty Russell 已提交
3731

3732
	dynamic_debug_setup(mod, info->debug, info->num_debug);
3733

3734 3735 3736
	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
	ftrace_module_init(mod);

3737 3738 3739
	/* Finally it's fully formed, ready to start executing. */
	err = complete_formation(mod, info);
	if (err)
3740
		goto ddebug_cleanup;
3741

3742 3743 3744 3745
	err = prepare_coming_module(mod);
	if (err)
		goto bug_cleanup;

3746
	/* Module is ready to execute: parsing args may do that. */
3747
	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
3748
				  -32768, 32767, mod,
3749
				  unknown_module_param_cb);
3750 3751
	if (IS_ERR(after_dashes)) {
		err = PTR_ERR(after_dashes);
3752
		goto coming_cleanup;
3753 3754 3755 3756
	} else if (after_dashes) {
		pr_warn("%s: parameters '%s' after `--' ignored\n",
		       mod->name, after_dashes);
	}
L
Linus Torvalds 已提交
3757

3758
	/* Link in to sysfs. */
3759
	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
L
Linus Torvalds 已提交
3760
	if (err < 0)
3761
		goto coming_cleanup;
3762

3763 3764 3765 3766 3767 3768
	if (is_livepatch_module(mod)) {
		err = copy_module_elf(mod, info);
		if (err < 0)
			goto sysfs_cleanup;
	}

3769
	/* Get rid of temporary copy. */
3770
	free_copy(info);
L
Linus Torvalds 已提交
3771 3772

	/* Done! */
3773
	trace_module_load(mod);
3774 3775

	return do_init_module(mod);
L
Linus Torvalds 已提交
3776

3777 3778
 sysfs_cleanup:
	mod_sysfs_teardown(mod);
3779
 coming_cleanup:
3780
	mod->state = MODULE_STATE_GOING;
3781
	destroy_params(mod->kp, mod->num_kp);
3782 3783
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
3784
	klp_module_going(mod);
3785 3786
 bug_cleanup:
	/* module_bug_cleanup needs module_mutex protection */
3787
	mutex_lock(&module_mutex);
3788
	module_bug_cleanup(mod);
3789
	mutex_unlock(&module_mutex);
3790 3791

	/* we can't deallocate the module until we clear memory protection */
R
Rusty Russell 已提交
3792 3793
	module_disable_ro(mod);
	module_disable_nx(mod);
3794

3795
 ddebug_cleanup:
3796
	dynamic_debug_remove(mod, info->debug);
3797
	synchronize_sched();
3798 3799
	kfree(mod->args);
 free_arch_cleanup:
L
Linus Torvalds 已提交
3800
	module_arch_cleanup(mod);
R
Rusty Russell 已提交
3801
 free_modinfo:
3802
	free_modinfo(mod);
3803
 free_unload:
L
Linus Torvalds 已提交
3804
	module_unload_free(mod);
3805 3806 3807 3808
 unlink_mod:
	mutex_lock(&module_mutex);
	/* Unlink carefully: kallsyms could be walking list. */
	list_del_rcu(&mod->list);
3809
	mod_tree_remove(mod);
3810
	wake_up_all(&module_wq);
3811 3812
	/* Wait for RCU-sched synchronizing before releasing mod->list. */
	synchronize_sched();
3813
	mutex_unlock(&module_mutex);
R
Rusty Russell 已提交
3814
 free_module:
3815 3816 3817 3818 3819 3820
	/*
	 * Ftrace needs to clean up what it initialized.
	 * This does nothing if ftrace_module_init() wasn't called,
	 * but it must be called outside of module_mutex.
	 */
	ftrace_release_mod(mod);
3821
	/* Free lock-classes; relies on the preceding sync_rcu() */
3822
	lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
3823

3824
	module_deallocate(mod, info);
R
Rusty Russell 已提交
3825
 free_copy:
3826 3827
	free_copy(info);
	return err;
3828 3829
}

3830 3831
SYSCALL_DEFINE3(init_module, void __user *, umod,
		unsigned long, len, const char __user *, uargs)
L
Linus Torvalds 已提交
3832
{
3833 3834
	int err;
	struct load_info info = { };
L
Linus Torvalds 已提交
3835

3836 3837 3838
	err = may_init_module();
	if (err)
		return err;
L
Linus Torvalds 已提交
3839

3840 3841
	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
	       umod, len, uargs);
L
Linus Torvalds 已提交
3842

3843 3844 3845
	err = copy_module_from_user(umod, len, &info);
	if (err)
		return err;
L
Linus Torvalds 已提交
3846

3847
	return load_module(&info, uargs, 0);
3848
}
3849

3850
SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3851 3852
{
	struct load_info info = { };
3853 3854 3855
	loff_t size;
	void *hdr;
	int err;
3856

3857 3858 3859
	err = may_init_module();
	if (err)
		return err;
L
Linus Torvalds 已提交
3860

3861
	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3862

3863 3864 3865
	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
		      |MODULE_INIT_IGNORE_VERMAGIC))
		return -EINVAL;
3866

3867 3868
	err = kernel_read_file_from_fd(fd, &hdr, &size, INT_MAX,
				       READING_MODULE);
3869 3870
	if (err)
		return err;
3871 3872
	info.hdr = hdr;
	info.len = size;
L
Linus Torvalds 已提交
3873

3874
	return load_module(&info, uargs, flags);
L
Linus Torvalds 已提交
3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888
}

static inline int within(unsigned long addr, void *start, unsigned long size)
{
	return ((void *)addr >= start && (void *)addr < start + size);
}

#ifdef CONFIG_KALLSYMS
/*
 * This ignores the intensely annoying "mapping symbols" found
 * in ARM ELF files: $a, $t and $d.
 */
static inline int is_arm_mapping_symbol(const char *str)
{
3889 3890
	if (str[0] == '.' && str[1] == 'L')
		return true;
K
Kyle McMartin 已提交
3891
	return str[0] == '$' && strchr("axtd", str[1])
L
Linus Torvalds 已提交
3892 3893 3894
	       && (str[2] == '\0' || str[2] == '.');
}

3895
static const char *symname(struct mod_kallsyms *kallsyms, unsigned int symnum)
R
Rusty Russell 已提交
3896
{
3897
	return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
R
Rusty Russell 已提交
3898 3899
}

L
Linus Torvalds 已提交
3900 3901 3902 3903 3904 3905 3906
static const char *get_ksymbol(struct module *mod,
			       unsigned long addr,
			       unsigned long *size,
			       unsigned long *offset)
{
	unsigned int i, best = 0;
	unsigned long nextval;
3907
	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
L
Linus Torvalds 已提交
3908 3909

	/* At worse, next value is at end of module */
3910
	if (within_module_init(addr, mod))
3911
		nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size;
D
Daniel Walker 已提交
3912
	else
3913
		nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
L
Linus Torvalds 已提交
3914

L
Lucas De Marchi 已提交
3915
	/* Scan for closest preceding symbol, and next symbol. (ELF
D
Daniel Walker 已提交
3916
	   starts real symbols at 1). */
3917 3918
	for (i = 1; i < kallsyms->num_symtab; i++) {
		if (kallsyms->symtab[i].st_shndx == SHN_UNDEF)
L
Linus Torvalds 已提交
3919 3920 3921 3922
			continue;

		/* We ignore unnamed symbols: they're uninformative
		 * and inserted at a whim. */
3923 3924
		if (*symname(kallsyms, i) == '\0'
		    || is_arm_mapping_symbol(symname(kallsyms, i)))
R
Rusty Russell 已提交
3925 3926
			continue;

3927 3928
		if (kallsyms->symtab[i].st_value <= addr
		    && kallsyms->symtab[i].st_value > kallsyms->symtab[best].st_value)
L
Linus Torvalds 已提交
3929
			best = i;
3930 3931 3932
		if (kallsyms->symtab[i].st_value > addr
		    && kallsyms->symtab[i].st_value < nextval)
			nextval = kallsyms->symtab[i].st_value;
L
Linus Torvalds 已提交
3933 3934 3935 3936 3937
	}

	if (!best)
		return NULL;

A
Alexey Dobriyan 已提交
3938
	if (size)
3939
		*size = nextval - kallsyms->symtab[best].st_value;
A
Alexey Dobriyan 已提交
3940
	if (offset)
3941 3942
		*offset = addr - kallsyms->symtab[best].st_value;
	return symname(kallsyms, best);
L
Linus Torvalds 已提交
3943 3944
}

3945 3946
/* For kallsyms to ask for address resolution.  NULL means not found.  Careful
 * not to lock to avoid deadlock on oopses, simply disable preemption. */
3947
const char *module_address_lookup(unsigned long addr,
3948 3949 3950 3951
			    unsigned long *size,
			    unsigned long *offset,
			    char **modname,
			    char *namebuf)
L
Linus Torvalds 已提交
3952
{
3953
	const char *ret = NULL;
3954
	struct module *mod;
L
Linus Torvalds 已提交
3955

3956
	preempt_disable();
3957 3958 3959 3960 3961
	mod = __module_address(addr);
	if (mod) {
		if (modname)
			*modname = mod->name;
		ret = get_ksymbol(mod, addr, size, offset);
L
Linus Torvalds 已提交
3962
	}
3963 3964 3965 3966 3967
	/* Make a copy in here where it's safe */
	if (ret) {
		strncpy(namebuf, ret, KSYM_NAME_LEN - 1);
		ret = namebuf;
	}
3968
	preempt_enable();
3969

3970
	return ret;
L
Linus Torvalds 已提交
3971 3972
}

3973 3974 3975 3976
int lookup_module_symbol_name(unsigned long addr, char *symname)
{
	struct module *mod;

3977
	preempt_disable();
3978
	list_for_each_entry_rcu(mod, &modules, list) {
3979 3980
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
3981
		if (within_module(addr, mod)) {
3982 3983 3984 3985 3986
			const char *sym;

			sym = get_ksymbol(mod, addr, NULL, NULL);
			if (!sym)
				goto out;
3987
			strlcpy(symname, sym, KSYM_NAME_LEN);
3988
			preempt_enable();
3989 3990 3991 3992
			return 0;
		}
	}
out:
3993
	preempt_enable();
3994 3995 3996
	return -ERANGE;
}

3997 3998 3999 4000 4001
int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
			unsigned long *offset, char *modname, char *name)
{
	struct module *mod;

4002
	preempt_disable();
4003
	list_for_each_entry_rcu(mod, &modules, list) {
4004 4005
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
4006
		if (within_module(addr, mod)) {
4007 4008 4009 4010 4011 4012
			const char *sym;

			sym = get_ksymbol(mod, addr, size, offset);
			if (!sym)
				goto out;
			if (modname)
4013
				strlcpy(modname, mod->name, MODULE_NAME_LEN);
4014
			if (name)
4015
				strlcpy(name, sym, KSYM_NAME_LEN);
4016
			preempt_enable();
4017 4018 4019 4020
			return 0;
		}
	}
out:
4021
	preempt_enable();
4022 4023 4024
	return -ERANGE;
}

4025 4026
int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
			char *name, char *module_name, int *exported)
L
Linus Torvalds 已提交
4027 4028 4029
{
	struct module *mod;

4030
	preempt_disable();
4031
	list_for_each_entry_rcu(mod, &modules, list) {
4032 4033
		struct mod_kallsyms *kallsyms;

4034 4035
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
4036 4037 4038 4039 4040
		kallsyms = rcu_dereference_sched(mod->kallsyms);
		if (symnum < kallsyms->num_symtab) {
			*value = kallsyms->symtab[symnum].st_value;
			*type = kallsyms->symtab[symnum].st_info;
			strlcpy(name, symname(kallsyms, symnum), KSYM_NAME_LEN);
4041
			strlcpy(module_name, mod->name, MODULE_NAME_LEN);
4042
			*exported = is_exported(name, *value, mod);
4043
			preempt_enable();
4044
			return 0;
L
Linus Torvalds 已提交
4045
		}
4046
		symnum -= kallsyms->num_symtab;
L
Linus Torvalds 已提交
4047
	}
4048
	preempt_enable();
4049
	return -ERANGE;
L
Linus Torvalds 已提交
4050 4051 4052 4053 4054
}

static unsigned long mod_find_symname(struct module *mod, const char *name)
{
	unsigned int i;
4055
	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
L
Linus Torvalds 已提交
4056

4057 4058 4059 4060
	for (i = 0; i < kallsyms->num_symtab; i++)
		if (strcmp(name, symname(kallsyms, i)) == 0 &&
		    kallsyms->symtab[i].st_info != 'U')
			return kallsyms->symtab[i].st_value;
L
Linus Torvalds 已提交
4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071
	return 0;
}

/* Look for this name: can be of form module:name. */
unsigned long module_kallsyms_lookup_name(const char *name)
{
	struct module *mod;
	char *colon;
	unsigned long ret = 0;

	/* Don't lock: we're in enough trouble already. */
4072
	preempt_disable();
4073
	if ((colon = strnchr(name, MODULE_NAME_LEN, ':')) != NULL) {
4074
		if ((mod = find_module_all(name, colon - name, false)) != NULL)
L
Linus Torvalds 已提交
4075 4076
			ret = mod_find_symname(mod, colon+1);
	} else {
4077 4078 4079
		list_for_each_entry_rcu(mod, &modules, list) {
			if (mod->state == MODULE_STATE_UNFORMED)
				continue;
L
Linus Torvalds 已提交
4080 4081
			if ((ret = mod_find_symname(mod, name)) != 0)
				break;
4082
		}
L
Linus Torvalds 已提交
4083
	}
4084
	preempt_enable();
L
Linus Torvalds 已提交
4085 4086
	return ret;
}
4087 4088 4089 4090 4091 4092 4093 4094 4095

int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
					     struct module *, unsigned long),
				   void *data)
{
	struct module *mod;
	unsigned int i;
	int ret;

4096 4097
	module_assert_mutex();

4098
	list_for_each_entry(mod, &modules, list) {
4099 4100 4101
		/* We hold module_mutex: no need for rcu_dereference_sched */
		struct mod_kallsyms *kallsyms = mod->kallsyms;

4102 4103
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
4104 4105 4106
		for (i = 0; i < kallsyms->num_symtab; i++) {
			ret = fn(data, symname(kallsyms, i),
				 mod, kallsyms->symtab[i].st_value);
4107 4108 4109 4110 4111 4112
			if (ret != 0)
				return ret;
		}
	}
	return 0;
}
L
Linus Torvalds 已提交
4113 4114
#endif /* CONFIG_KALLSYMS */

4115 4116 4117 4118
/* Maximum number of characters written by module_flags() */
#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)

/* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
4119
static char *module_flags(struct module *mod, char *buf)
4120 4121 4122
{
	int bx = 0;

4123
	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
4124 4125 4126
	if (mod->taints ||
	    mod->state == MODULE_STATE_GOING ||
	    mod->state == MODULE_STATE_COMING) {
4127
		buf[bx++] = '(';
4128
		bx += module_flags_taint(mod, buf + bx);
4129 4130 4131 4132 4133 4134
		/* Show a - for module-is-being-unloaded */
		if (mod->state == MODULE_STATE_GOING)
			buf[bx++] = '-';
		/* Show a + for module-is-being-loaded */
		if (mod->state == MODULE_STATE_COMING)
			buf[bx++] = '+';
4135 4136 4137 4138 4139 4140 4141
		buf[bx++] = ')';
	}
	buf[bx] = '\0';

	return buf;
}

4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159
#ifdef CONFIG_PROC_FS
/* Called by the /proc file system to return a list of modules. */
static void *m_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&module_mutex);
	return seq_list_start(&modules, *pos);
}

static void *m_next(struct seq_file *m, void *p, loff_t *pos)
{
	return seq_list_next(p, &modules, pos);
}

static void m_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&module_mutex);
}

L
Linus Torvalds 已提交
4160 4161 4162
static int m_show(struct seq_file *m, void *p)
{
	struct module *mod = list_entry(p, struct module, list);
4163
	char buf[MODULE_FLAGS_BUF_SIZE];
4164
	void *value;
4165

4166 4167 4168 4169
	/* We always ignore unformed modules. */
	if (mod->state == MODULE_STATE_UNFORMED)
		return 0;

4170
	seq_printf(m, "%s %u",
4171
		   mod->name, mod->init_layout.size + mod->core_layout.size);
L
Linus Torvalds 已提交
4172 4173 4174 4175
	print_unload_info(m, mod);

	/* Informative for users. */
	seq_printf(m, " %s",
4176 4177
		   mod->state == MODULE_STATE_GOING ? "Unloading" :
		   mod->state == MODULE_STATE_COMING ? "Loading" :
L
Linus Torvalds 已提交
4178 4179
		   "Live");
	/* Used by oprofile and other similar tools. */
4180 4181
	value = m->private ? NULL : mod->core_layout.base;
	seq_printf(m, " 0x%px", value);
L
Linus Torvalds 已提交
4182

4183 4184
	/* Taints info */
	if (mod->taints)
4185
		seq_printf(m, " %s", module_flags(mod, buf));
4186

4187
	seq_puts(m, "\n");
L
Linus Torvalds 已提交
4188 4189 4190 4191 4192 4193 4194 4195
	return 0;
}

/* Format: modulename size refcount deps address

   Where refcount is a number or -, and deps is a comma-separated list
   of depends or -.
*/
4196
static const struct seq_operations modules_op = {
L
Linus Torvalds 已提交
4197 4198 4199 4200 4201 4202
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= m_show
};

4203 4204 4205 4206 4207 4208 4209
/*
 * This also sets the "private" pointer to non-NULL if the
 * kernel pointers should be hidden (so you can just test
 * "m->private" to see if you should keep the values private).
 *
 * We use the same logic as for /proc/kallsyms.
 */
4210 4211
static int modules_open(struct inode *inode, struct file *file)
{
4212 4213 4214 4215 4216 4217 4218 4219
	int err = seq_open(file, &modules_op);

	if (!err) {
		struct seq_file *m = file->private_data;
		m->private = kallsyms_show_value() ? NULL : (void *)8ul;
	}

	return 0;
4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236
}

static const struct file_operations proc_modules_operations = {
	.open		= modules_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init proc_modules_init(void)
{
	proc_create("modules", 0, NULL, &proc_modules_operations);
	return 0;
}
module_init(proc_modules_init);
#endif

L
Linus Torvalds 已提交
4237 4238 4239 4240 4241 4242
/* Given an address, look for it in the module exception tables. */
const struct exception_table_entry *search_module_extables(unsigned long addr)
{
	const struct exception_table_entry *e = NULL;
	struct module *mod;

R
Rusty Russell 已提交
4243
	preempt_disable();
4244 4245 4246
	mod = __module_address(addr);
	if (!mod)
		goto out;
D
Daniel Walker 已提交
4247

4248 4249 4250 4251
	if (!mod->num_exentries)
		goto out;

	e = search_extable(mod->extable,
4252
			   mod->num_exentries,
4253 4254
			   addr);
out:
R
Rusty Russell 已提交
4255
	preempt_enable();
L
Linus Torvalds 已提交
4256

4257 4258 4259 4260
	/*
	 * Now, if we found one, we are running inside it now, hence
	 * we cannot unload the module, hence no refcnt needed.
	 */
L
Linus Torvalds 已提交
4261 4262 4263
	return e;
}

4264
/*
R
Rusty Russell 已提交
4265 4266 4267 4268 4269
 * is_module_address - is this address inside a module?
 * @addr: the address to check.
 *
 * See is_module_text_address() if you simply want to see if the address
 * is code (not data).
4270
 */
R
Rusty Russell 已提交
4271
bool is_module_address(unsigned long addr)
4272
{
R
Rusty Russell 已提交
4273
	bool ret;
4274

R
Rusty Russell 已提交
4275
	preempt_disable();
R
Rusty Russell 已提交
4276
	ret = __module_address(addr) != NULL;
R
Rusty Russell 已提交
4277
	preempt_enable();
4278

R
Rusty Russell 已提交
4279
	return ret;
4280 4281
}

R
Rusty Russell 已提交
4282 4283 4284 4285 4286 4287 4288
/*
 * __module_address - get the module which contains an address.
 * @addr: the address.
 *
 * Must be called with preempt disabled or module mutex held so that
 * module doesn't get freed during this.
 */
4289
struct module *__module_address(unsigned long addr)
L
Linus Torvalds 已提交
4290 4291 4292
{
	struct module *mod;

4293 4294 4295
	if (addr < module_addr_min || addr > module_addr_max)
		return NULL;

4296 4297
	module_assert_mutex_or_preempt();

4298
	mod = mod_find(addr);
4299 4300
	if (mod) {
		BUG_ON(!within_module(addr, mod));
4301
		if (mod->state == MODULE_STATE_UNFORMED)
4302
			mod = NULL;
4303
	}
4304
	return mod;
L
Linus Torvalds 已提交
4305
}
4306
EXPORT_SYMBOL_GPL(__module_address);
L
Linus Torvalds 已提交
4307

R
Rusty Russell 已提交
4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338
/*
 * is_module_text_address - is this address inside module code?
 * @addr: the address to check.
 *
 * See is_module_address() if you simply want to see if the address is
 * anywhere in a module.  See kernel_text_address() for testing if an
 * address corresponds to kernel or module code.
 */
bool is_module_text_address(unsigned long addr)
{
	bool ret;

	preempt_disable();
	ret = __module_text_address(addr) != NULL;
	preempt_enable();

	return ret;
}

/*
 * __module_text_address - get the module whose code contains an address.
 * @addr: the address.
 *
 * Must be called with preempt disabled or module mutex held so that
 * module doesn't get freed during this.
 */
struct module *__module_text_address(unsigned long addr)
{
	struct module *mod = __module_address(addr);
	if (mod) {
		/* Make sure it's within the text section. */
4339 4340
		if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
		    && !within(addr, mod->core_layout.base, mod->core_layout.text_size))
R
Rusty Russell 已提交
4341 4342 4343 4344
			mod = NULL;
	}
	return mod;
}
4345
EXPORT_SYMBOL_GPL(__module_text_address);
R
Rusty Russell 已提交
4346

L
Linus Torvalds 已提交
4347 4348 4349 4350
/* Don't grab lock, we're oopsing. */
void print_modules(void)
{
	struct module *mod;
4351
	char buf[MODULE_FLAGS_BUF_SIZE];
L
Linus Torvalds 已提交
4352

4353
	printk(KERN_DEFAULT "Modules linked in:");
4354 4355
	/* Most callers should already have preempt disabled, but make sure */
	preempt_disable();
4356 4357 4358
	list_for_each_entry_rcu(mod, &modules, list) {
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
J
Jiri Slaby 已提交
4359
		pr_cont(" %s%s", mod->name, module_flags(mod, buf));
4360
	}
4361
	preempt_enable();
4362
	if (last_unloaded_module[0])
J
Jiri Slaby 已提交
4363 4364
		pr_cont(" [last unloaded: %s]", last_unloaded_module);
	pr_cont("\n");
L
Linus Torvalds 已提交
4365 4366 4367
}

#ifdef CONFIG_MODVERSIONS
4368 4369 4370 4371 4372 4373
/* Generate the signature for all relevant module structures here.
 * If these change, we don't want to try to parse the module. */
void module_layout(struct module *mod,
		   struct modversion_info *ver,
		   struct kernel_param *kp,
		   struct kernel_symbol *ks,
4374
		   struct tracepoint * const *tp)
4375 4376 4377
{
}
EXPORT_SYMBOL(module_layout);
L
Linus Torvalds 已提交
4378
#endif