module.c 104.3 KB
Newer Older
1
/*
L
Linus Torvalds 已提交
2
   Copyright (C) 2002 Richard Henderson
3
   Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/
19
#include <linux/export.h>
L
Linus Torvalds 已提交
20
#include <linux/moduleloader.h>
21
#include <linux/trace_events.h>
L
Linus Torvalds 已提交
22
#include <linux/init.h>
23
#include <linux/kallsyms.h>
24
#include <linux/file.h>
25
#include <linux/fs.h>
R
Roland McGrath 已提交
26
#include <linux/sysfs.h>
27
#include <linux/kernel.h>
L
Linus Torvalds 已提交
28 29 30
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/elf.h>
31
#include <linux/proc_fs.h>
32
#include <linux/security.h>
L
Linus Torvalds 已提交
33 34 35 36
#include <linux/seq_file.h>
#include <linux/syscalls.h>
#include <linux/fcntl.h>
#include <linux/rcupdate.h>
37
#include <linux/capability.h>
L
Linus Torvalds 已提交
38 39 40 41 42 43
#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/vermagic.h>
#include <linux/notifier.h>
A
Al Viro 已提交
44
#include <linux/sched.h>
L
Linus Torvalds 已提交
45
#include <linux/device.h>
46
#include <linux/string.h>
A
Arjan van de Ven 已提交
47
#include <linux/mutex.h>
48
#include <linux/rculist.h>
L
Linus Torvalds 已提交
49 50
#include <asm/uaccess.h>
#include <asm/cacheflush.h>
51
#include <asm/mmu_context.h>
52
#include <linux/license.h>
53
#include <asm/sections.h>
M
Mathieu Desnoyers 已提交
54
#include <linux/tracepoint.h>
55
#include <linux/ftrace.h>
56
#include <linux/async.h>
57
#include <linux/percpu.h>
C
Catalin Marinas 已提交
58
#include <linux/kmemleak.h>
59
#include <linux/jump_label.h>
60
#include <linux/pfn.h>
61
#include <linux/bsearch.h>
62
#include <uapi/linux/module.h>
R
Rusty Russell 已提交
63
#include "module-internal.h"
L
Linus Torvalds 已提交
64

65 66 67
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>

L
Linus Torvalds 已提交
68 69 70 71
#ifndef ARCH_SHF_SMALL
#define ARCH_SHF_SMALL 0
#endif

72 73 74 75 76 77 78 79 80 81 82
/*
 * Modules' sections will be aligned on page boundaries
 * to ensure complete separation of code and data, but
 * only when CONFIG_DEBUG_SET_MODULE_RONX=y
 */
#ifdef CONFIG_DEBUG_SET_MODULE_RONX
# define debug_align(X) ALIGN(X, PAGE_SIZE)
#else
# define debug_align(X) (X)
#endif

L
Linus Torvalds 已提交
83 84 85
/* If this is set, the section belongs in the init part of the module */
#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))

86 87 88 89 90
/*
 * Mutex protects:
 * 1) List of modules (also safely readable with preempt_disable),
 * 2) module_use links,
 * 3) module_addr_min/module_addr_max.
91
 * (delete and add uses RCU list operations). */
92 93
DEFINE_MUTEX(module_mutex);
EXPORT_SYMBOL_GPL(module_mutex);
L
Linus Torvalds 已提交
94
static LIST_HEAD(modules);
95

96
#ifdef CONFIG_MODULES_TREE_LOOKUP
R
Rusty Russell 已提交
97

98 99 100 101
/*
 * Use a latched RB-tree for __module_address(); this allows us to use
 * RCU-sched lookups of the address from any context.
 *
102 103 104
 * This is conditional on PERF_EVENTS || TRACING because those can really hit
 * __module_address() hard by doing a lot of stack unwinding; potentially from
 * NMI context.
105 106 107
 */

static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
R
Rusty Russell 已提交
108
{
109
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
R
Rusty Russell 已提交
110

111
	return (unsigned long)layout->base;
112 113 114 115
}

static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
{
116
	struct module_layout *layout = container_of(n, struct module_layout, mtn.node);
117

118
	return (unsigned long)layout->size;
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
}

static __always_inline bool
mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
{
	return __mod_tree_val(a) < __mod_tree_val(b);
}

static __always_inline int
mod_tree_comp(void *key, struct latch_tree_node *n)
{
	unsigned long val = (unsigned long)key;
	unsigned long start, end;

	start = __mod_tree_val(n);
	if (val < start)
		return -1;

	end = start + __mod_tree_size(n);
	if (val >= end)
		return 1;
R
Rusty Russell 已提交
140 141 142 143

	return 0;
}

144 145 146 147 148
static const struct latch_tree_ops mod_tree_ops = {
	.less = mod_tree_less,
	.comp = mod_tree_comp,
};

149 150 151 152 153 154
static struct mod_tree_root {
	struct latch_tree_root root;
	unsigned long addr_min;
	unsigned long addr_max;
} mod_tree __cacheline_aligned = {
	.addr_min = -1UL,
R
Rusty Russell 已提交
155 156
};

157 158 159 160 161 162 163 164 165 166 167 168
#define module_addr_min mod_tree.addr_min
#define module_addr_max mod_tree.addr_max

static noinline void __mod_tree_insert(struct mod_tree_node *node)
{
	latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
}

static void __mod_tree_remove(struct mod_tree_node *node)
{
	latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
}
169 170 171 172 173 174 175

/*
 * These modifications: insert, remove_init and remove; are serialized by the
 * module_mutex.
 */
static void mod_tree_insert(struct module *mod)
{
176 177
	mod->core_layout.mtn.mod = mod;
	mod->init_layout.mtn.mod = mod;
178

179 180 181
	__mod_tree_insert(&mod->core_layout.mtn);
	if (mod->init_layout.size)
		__mod_tree_insert(&mod->init_layout.mtn);
182 183 184 185
}

static void mod_tree_remove_init(struct module *mod)
{
186 187
	if (mod->init_layout.size)
		__mod_tree_remove(&mod->init_layout.mtn);
188 189 190 191
}

static void mod_tree_remove(struct module *mod)
{
192
	__mod_tree_remove(&mod->core_layout.mtn);
193 194 195
	mod_tree_remove_init(mod);
}

196
static struct module *mod_find(unsigned long addr)
197 198 199
{
	struct latch_tree_node *ltn;

200
	ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
201 202 203 204 205 206
	if (!ltn)
		return NULL;

	return container_of(ltn, struct mod_tree_node, node)->mod;
}

207 208
#else /* MODULES_TREE_LOOKUP */

209 210
static unsigned long module_addr_min = -1UL, module_addr_max = 0;

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228
static void mod_tree_insert(struct module *mod) { }
static void mod_tree_remove_init(struct module *mod) { }
static void mod_tree_remove(struct module *mod) { }

static struct module *mod_find(unsigned long addr)
{
	struct module *mod;

	list_for_each_entry_rcu(mod, &modules, list) {
		if (within_module(addr, mod))
			return mod;
	}

	return NULL;
}

#endif /* MODULES_TREE_LOOKUP */

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
/*
 * Bounds of module text, for speeding up __module_address.
 * Protected by module_mutex.
 */
static void __mod_update_bounds(void *base, unsigned int size)
{
	unsigned long min = (unsigned long)base;
	unsigned long max = min + size;

	if (min < module_addr_min)
		module_addr_min = min;
	if (max > module_addr_max)
		module_addr_max = max;
}

static void mod_update_bounds(struct module *mod)
{
246 247 248
	__mod_update_bounds(mod->core_layout.base, mod->core_layout.size);
	if (mod->init_layout.size)
		__mod_update_bounds(mod->init_layout.base, mod->init_layout.size);
249 250
}

251 252 253 254
#ifdef CONFIG_KGDB_KDB
struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
#endif /* CONFIG_KGDB_KDB */

255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
static void module_assert_mutex(void)
{
	lockdep_assert_held(&module_mutex);
}

static void module_assert_mutex_or_preempt(void)
{
#ifdef CONFIG_LOCKDEP
	if (unlikely(!debug_locks))
		return;

	WARN_ON(!rcu_read_lock_sched_held() &&
		!lockdep_is_held(&module_mutex));
#endif
}

271 272
static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
#ifndef CONFIG_MODULE_SIG_FORCE
R
Rusty Russell 已提交
273 274
module_param(sig_enforce, bool_enable_only, 0644);
#endif /* !CONFIG_MODULE_SIG_FORCE */
L
Linus Torvalds 已提交
275

276 277
/* Block module loading/unloading? */
int modules_disabled = 0;
278
core_param(nomodule, modules_disabled, bint, 0);
279

280 281 282
/* Waiting for a module to finish initializing? */
static DECLARE_WAIT_QUEUE_HEAD(module_wq);

283
static BLOCKING_NOTIFIER_HEAD(module_notify_list);
L
Linus Torvalds 已提交
284

285
int register_module_notifier(struct notifier_block *nb)
L
Linus Torvalds 已提交
286
{
287
	return blocking_notifier_chain_register(&module_notify_list, nb);
L
Linus Torvalds 已提交
288 289 290
}
EXPORT_SYMBOL(register_module_notifier);

291
int unregister_module_notifier(struct notifier_block *nb)
L
Linus Torvalds 已提交
292
{
293
	return blocking_notifier_chain_unregister(&module_notify_list, nb);
L
Linus Torvalds 已提交
294 295 296
}
EXPORT_SYMBOL(unregister_module_notifier);

297 298 299 300
struct load_info {
	Elf_Ehdr *hdr;
	unsigned long len;
	Elf_Shdr *sechdrs;
301
	char *secstrings, *strtab;
R
Rusty Russell 已提交
302
	unsigned long symoffs, stroffs;
303 304
	struct _ddebug *debug;
	unsigned int num_debug;
R
Rusty Russell 已提交
305
	bool sig_ok;
306 307 308
#ifdef CONFIG_KALLSYMS
	unsigned long mod_kallsyms_init_off;
#endif
309 310 311 312 313
	struct {
		unsigned int sym, str, mod, vers, info, pcpu;
	} index;
};

314 315
/* We require a truly strong try_module_get(): 0 means failure due to
   ongoing or failed initialization etc. */
L
Linus Torvalds 已提交
316 317
static inline int strong_try_module_get(struct module *mod)
{
318
	BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
L
Linus Torvalds 已提交
319
	if (mod && mod->state == MODULE_STATE_COMING)
320 321
		return -EBUSY;
	if (try_module_get(mod))
L
Linus Torvalds 已提交
322
		return 0;
323 324
	else
		return -ENOENT;
L
Linus Torvalds 已提交
325 326
}

327 328
static inline void add_taint_module(struct module *mod, unsigned flag,
				    enum lockdep_ok lockdep_ok)
329
{
330
	add_taint(flag, lockdep_ok);
A
Andi Kleen 已提交
331
	mod->taints |= (1U << flag);
332 333
}

334 335 336
/*
 * A thread that wants to hold a reference to a module only while it
 * is running can call this to safely exit.  nfsd and lockd use this.
L
Linus Torvalds 已提交
337 338 339 340 341 342 343
 */
void __module_put_and_exit(struct module *mod, long code)
{
	module_put(mod);
	do_exit(code);
}
EXPORT_SYMBOL(__module_put_and_exit);
D
Daniel Walker 已提交
344

L
Linus Torvalds 已提交
345
/* Find a module section: 0 means not found. */
346
static unsigned int find_sec(const struct load_info *info, const char *name)
L
Linus Torvalds 已提交
347 348 349
{
	unsigned int i;

350 351
	for (i = 1; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *shdr = &info->sechdrs[i];
L
Linus Torvalds 已提交
352
		/* Alloc bit cleared means "ignore it." */
353 354
		if ((shdr->sh_flags & SHF_ALLOC)
		    && strcmp(info->secstrings + shdr->sh_name, name) == 0)
L
Linus Torvalds 已提交
355
			return i;
356
	}
L
Linus Torvalds 已提交
357 358 359
	return 0;
}

R
Rusty Russell 已提交
360
/* Find a module section, or NULL. */
361
static void *section_addr(const struct load_info *info, const char *name)
R
Rusty Russell 已提交
362 363
{
	/* Section 0 has sh_addr 0. */
364
	return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
R
Rusty Russell 已提交
365 366 367
}

/* Find a module section, or NULL.  Fill in number of "objects" in section. */
368
static void *section_objs(const struct load_info *info,
R
Rusty Russell 已提交
369 370 371 372
			  const char *name,
			  size_t object_size,
			  unsigned int *num)
{
373
	unsigned int sec = find_sec(info, name);
R
Rusty Russell 已提交
374 375

	/* Section 0 has sh_addr 0 and sh_size 0. */
376 377
	*num = info->sechdrs[sec].sh_size / object_size;
	return (void *)info->sechdrs[sec].sh_addr;
R
Rusty Russell 已提交
378 379
}

L
Linus Torvalds 已提交
380 381 382 383 384
/* Provided by the linker */
extern const struct kernel_symbol __start___ksymtab[];
extern const struct kernel_symbol __stop___ksymtab[];
extern const struct kernel_symbol __start___ksymtab_gpl[];
extern const struct kernel_symbol __stop___ksymtab_gpl[];
385 386
extern const struct kernel_symbol __start___ksymtab_gpl_future[];
extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
L
Linus Torvalds 已提交
387 388
extern const unsigned long __start___kcrctab[];
extern const unsigned long __start___kcrctab_gpl[];
389
extern const unsigned long __start___kcrctab_gpl_future[];
390 391 392 393 394
#ifdef CONFIG_UNUSED_SYMBOLS
extern const struct kernel_symbol __start___ksymtab_unused[];
extern const struct kernel_symbol __stop___ksymtab_unused[];
extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
395 396
extern const unsigned long __start___kcrctab_unused[];
extern const unsigned long __start___kcrctab_unused_gpl[];
397
#endif
L
Linus Torvalds 已提交
398 399 400 401

#ifndef CONFIG_MODVERSIONS
#define symversion(base, idx) NULL
#else
A
Andrew Morton 已提交
402
#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
L
Linus Torvalds 已提交
403 404
#endif

405 406 407 408 409
static bool each_symbol_in_section(const struct symsearch *arr,
				   unsigned int arrsize,
				   struct module *owner,
				   bool (*fn)(const struct symsearch *syms,
					      struct module *owner,
410
					      void *data),
411
				   void *data)
412
{
413
	unsigned int j;
414

415
	for (j = 0; j < arrsize; j++) {
416 417
		if (fn(&arr[j], owner, data))
			return true;
418
	}
419 420

	return false;
421 422
}

423
/* Returns true as soon as fn returns true, otherwise false. */
424 425 426 427
bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
				    struct module *owner,
				    void *data),
			 void *data)
428 429
{
	struct module *mod;
430
	static const struct symsearch arr[] = {
431
		{ __start___ksymtab, __stop___ksymtab, __start___kcrctab,
432
		  NOT_GPL_ONLY, false },
433
		{ __start___ksymtab_gpl, __stop___ksymtab_gpl,
434 435
		  __start___kcrctab_gpl,
		  GPL_ONLY, false },
436
		{ __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
437 438
		  __start___kcrctab_gpl_future,
		  WILL_BE_GPL_ONLY, false },
439
#ifdef CONFIG_UNUSED_SYMBOLS
440
		{ __start___ksymtab_unused, __stop___ksymtab_unused,
441 442
		  __start___kcrctab_unused,
		  NOT_GPL_ONLY, true },
443
		{ __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
444 445
		  __start___kcrctab_unused_gpl,
		  GPL_ONLY, true },
446
#endif
447
	};
448

449 450
	module_assert_mutex_or_preempt();

451 452
	if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
		return true;
453

454
	list_for_each_entry_rcu(mod, &modules, list) {
455 456
		struct symsearch arr[] = {
			{ mod->syms, mod->syms + mod->num_syms, mod->crcs,
457
			  NOT_GPL_ONLY, false },
458
			{ mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
459 460
			  mod->gpl_crcs,
			  GPL_ONLY, false },
461 462
			{ mod->gpl_future_syms,
			  mod->gpl_future_syms + mod->num_gpl_future_syms,
463 464
			  mod->gpl_future_crcs,
			  WILL_BE_GPL_ONLY, false },
465
#ifdef CONFIG_UNUSED_SYMBOLS
466 467
			{ mod->unused_syms,
			  mod->unused_syms + mod->num_unused_syms,
468 469
			  mod->unused_crcs,
			  NOT_GPL_ONLY, true },
470 471
			{ mod->unused_gpl_syms,
			  mod->unused_gpl_syms + mod->num_unused_gpl_syms,
472 473
			  mod->unused_gpl_crcs,
			  GPL_ONLY, true },
474
#endif
475 476
		};

477 478 479
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;

480 481 482 483 484
		if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
			return true;
	}
	return false;
}
485
EXPORT_SYMBOL_GPL(each_symbol_section);
486 487 488 489 490 491 492 493 494 495

struct find_symbol_arg {
	/* Input */
	const char *name;
	bool gplok;
	bool warn;

	/* Output */
	struct module *owner;
	const unsigned long *crc;
496
	const struct kernel_symbol *sym;
497 498
};

499 500 501
static bool check_symbol(const struct symsearch *syms,
				 struct module *owner,
				 unsigned int symnum, void *data)
502 503 504 505 506 507 508
{
	struct find_symbol_arg *fsa = data;

	if (!fsa->gplok) {
		if (syms->licence == GPL_ONLY)
			return false;
		if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
A
Andrew Morton 已提交
509 510 511
			pr_warn("Symbol %s is being used by a non-GPL module, "
				"which will not be allowed in the future\n",
				fsa->name);
512
		}
L
Linus Torvalds 已提交
513
	}
514

515
#ifdef CONFIG_UNUSED_SYMBOLS
516
	if (syms->unused && fsa->warn) {
A
Andrew Morton 已提交
517 518 519
		pr_warn("Symbol %s is marked as UNUSED, however this module is "
			"using it.\n", fsa->name);
		pr_warn("This symbol will go away in the future.\n");
520 521 522
		pr_warn("Please evaluate if this is the right api to use and "
			"if it really is, submit a report to the linux kernel "
			"mailing list together with submitting your code for "
A
Andrew Morton 已提交
523
			"inclusion.\n");
524
	}
525
#endif
526 527 528

	fsa->owner = owner;
	fsa->crc = symversion(syms->crcs, symnum);
529
	fsa->sym = &syms->start[symnum];
530 531 532
	return true;
}

533 534 535 536 537 538 539 540
static int cmp_name(const void *va, const void *vb)
{
	const char *a;
	const struct kernel_symbol *b;
	a = va; b = vb;
	return strcmp(a, b->name);
}

541 542 543 544 545
static bool find_symbol_in_section(const struct symsearch *syms,
				   struct module *owner,
				   void *data)
{
	struct find_symbol_arg *fsa = data;
546 547 548 549 550 551 552
	struct kernel_symbol *sym;

	sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
			sizeof(struct kernel_symbol), cmp_name);

	if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
		return true;
553 554 555 556

	return false;
}

557
/* Find a symbol and return it, along with, (optional) crc and
558
 * (optional) module which owns it.  Needs preempt disabled or module_mutex. */
559 560 561 562 563
const struct kernel_symbol *find_symbol(const char *name,
					struct module **owner,
					const unsigned long **crc,
					bool gplok,
					bool warn)
564 565 566 567 568 569 570
{
	struct find_symbol_arg fsa;

	fsa.name = name;
	fsa.gplok = gplok;
	fsa.warn = warn;

571
	if (each_symbol_section(find_symbol_in_section, &fsa)) {
572 573 574 575
		if (owner)
			*owner = fsa.owner;
		if (crc)
			*crc = fsa.crc;
576
		return fsa.sym;
577 578
	}

579
	pr_debug("Failed to find symbol %s\n", name);
580
	return NULL;
L
Linus Torvalds 已提交
581
}
582
EXPORT_SYMBOL_GPL(find_symbol);
L
Linus Torvalds 已提交
583

584 585 586 587
/*
 * Search for module by name: must hold module_mutex (or preempt disabled
 * for read-only access).
 */
588
static struct module *find_module_all(const char *name, size_t len,
589
				      bool even_unformed)
L
Linus Torvalds 已提交
590 591 592
{
	struct module *mod;

593
	module_assert_mutex_or_preempt();
594

L
Linus Torvalds 已提交
595
	list_for_each_entry(mod, &modules, list) {
596 597
		if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
			continue;
598
		if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
L
Linus Torvalds 已提交
599 600 601 602
			return mod;
	}
	return NULL;
}
603 604 605

struct module *find_module(const char *name)
{
606
	module_assert_mutex();
607
	return find_module_all(name, strlen(name), false);
608
}
609
EXPORT_SYMBOL_GPL(find_module);
L
Linus Torvalds 已提交
610 611

#ifdef CONFIG_SMP
612

613
static inline void __percpu *mod_percpu(struct module *mod)
614
{
615 616
	return mod->percpu;
}
617

R
Rusty Russell 已提交
618
static int percpu_modalloc(struct module *mod, struct load_info *info)
619
{
R
Rusty Russell 已提交
620 621 622 623 624 625
	Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
	unsigned long align = pcpusec->sh_addralign;

	if (!pcpusec->sh_size)
		return 0;

626
	if (align > PAGE_SIZE) {
A
Andrew Morton 已提交
627 628
		pr_warn("%s: per-cpu alignment %li > %li\n",
			mod->name, align, PAGE_SIZE);
629 630 631
		align = PAGE_SIZE;
	}

R
Rusty Russell 已提交
632
	mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
633
	if (!mod->percpu) {
A
Andrew Morton 已提交
634 635
		pr_warn("%s: Could not allocate %lu bytes percpu data\n",
			mod->name, (unsigned long)pcpusec->sh_size);
636 637
		return -ENOMEM;
	}
R
Rusty Russell 已提交
638
	mod->percpu_size = pcpusec->sh_size;
639
	return 0;
640 641
}

642
static void percpu_modfree(struct module *mod)
643
{
644
	free_percpu(mod->percpu);
645 646
}

647
static unsigned int find_pcpusec(struct load_info *info)
648
{
649
	return find_sec(info, ".data..percpu");
650 651
}

652 653
static void percpu_modcopy(struct module *mod,
			   const void *from, unsigned long size)
654 655 656 657
{
	int cpu;

	for_each_possible_cpu(cpu)
658
		memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
659 660
}

661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677
/**
 * is_module_percpu_address - test whether address is from module static percpu
 * @addr: address to test
 *
 * Test whether @addr belongs to module static percpu area.
 *
 * RETURNS:
 * %true if @addr is from module static percpu area
 */
bool is_module_percpu_address(unsigned long addr)
{
	struct module *mod;
	unsigned int cpu;

	preempt_disable();

	list_for_each_entry_rcu(mod, &modules, list) {
678 679
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
		if (!mod->percpu_size)
			continue;
		for_each_possible_cpu(cpu) {
			void *start = per_cpu_ptr(mod->percpu, cpu);

			if ((void *)addr >= start &&
			    (void *)addr < start + mod->percpu_size) {
				preempt_enable();
				return true;
			}
		}
	}

	preempt_enable();
	return false;
695 696
}

L
Linus Torvalds 已提交
697
#else /* ... !CONFIG_SMP */
698

699
static inline void __percpu *mod_percpu(struct module *mod)
L
Linus Torvalds 已提交
700 701 702
{
	return NULL;
}
R
Rusty Russell 已提交
703
static int percpu_modalloc(struct module *mod, struct load_info *info)
704
{
R
Rusty Russell 已提交
705 706 707 708
	/* UP modules shouldn't have this section: ENOMEM isn't quite right */
	if (info->sechdrs[info->index.pcpu].sh_size != 0)
		return -ENOMEM;
	return 0;
709 710
}
static inline void percpu_modfree(struct module *mod)
L
Linus Torvalds 已提交
711 712
{
}
713
static unsigned int find_pcpusec(struct load_info *info)
L
Linus Torvalds 已提交
714 715 716
{
	return 0;
}
717 718
static inline void percpu_modcopy(struct module *mod,
				  const void *from, unsigned long size)
L
Linus Torvalds 已提交
719 720 721 722
{
	/* pcpusec should be 0, and size of that section should be 0. */
	BUG_ON(size != 0);
}
723 724 725 726
bool is_module_percpu_address(unsigned long addr)
{
	return false;
}
727

L
Linus Torvalds 已提交
728 729
#endif /* CONFIG_SMP */

730 731 732 733 734 735
#define MODINFO_ATTR(field)	\
static void setup_modinfo_##field(struct module *mod, const char *s)  \
{                                                                     \
	mod->field = kstrdup(s, GFP_KERNEL);                          \
}                                                                     \
static ssize_t show_modinfo_##field(struct module_attribute *mattr,   \
736
			struct module_kobject *mk, char *buffer)      \
737
{                                                                     \
738
	return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field);  \
739 740 741 742 743 744 745
}                                                                     \
static int modinfo_##field##_exists(struct module *mod)               \
{                                                                     \
	return mod->field != NULL;                                    \
}                                                                     \
static void free_modinfo_##field(struct module *mod)                  \
{                                                                     \
D
Daniel Walker 已提交
746 747
	kfree(mod->field);                                            \
	mod->field = NULL;                                            \
748 749
}                                                                     \
static struct module_attribute modinfo_##field = {                    \
750
	.attr = { .name = __stringify(field), .mode = 0444 },         \
751 752 753 754 755 756 757 758 759
	.show = show_modinfo_##field,                                 \
	.setup = setup_modinfo_##field,                               \
	.test = modinfo_##field##_exists,                             \
	.free = free_modinfo_##field,                                 \
};

MODINFO_ATTR(version);
MODINFO_ATTR(srcversion);

760 761
static char last_unloaded_module[MODULE_NAME_LEN+1];

762
#ifdef CONFIG_MODULE_UNLOAD
763 764 765

EXPORT_TRACEPOINT_SYMBOL(module_get);

766 767 768
/* MODULE_REF_BASE is the base reference count by kmodule loader. */
#define MODULE_REF_BASE	1

L
Linus Torvalds 已提交
769
/* Init the unload section of the module. */
770
static int module_unload_init(struct module *mod)
L
Linus Torvalds 已提交
771
{
772 773 774 775 776
	/*
	 * Initialize reference counter to MODULE_REF_BASE.
	 * refcnt == 0 means module is going.
	 */
	atomic_set(&mod->refcnt, MODULE_REF_BASE);
777

778 779
	INIT_LIST_HEAD(&mod->source_list);
	INIT_LIST_HEAD(&mod->target_list);
780

L
Linus Torvalds 已提交
781
	/* Hold reference count during initialization. */
782
	atomic_inc(&mod->refcnt);
783 784

	return 0;
L
Linus Torvalds 已提交
785 786 787 788 789 790 791
}

/* Does a already use b? */
static int already_uses(struct module *a, struct module *b)
{
	struct module_use *use;

792 793
	list_for_each_entry(use, &b->source_list, source_list) {
		if (use->source == a) {
794
			pr_debug("%s uses %s!\n", a->name, b->name);
L
Linus Torvalds 已提交
795 796 797
			return 1;
		}
	}
798
	pr_debug("%s does not use %s!\n", a->name, b->name);
L
Linus Torvalds 已提交
799 800 801
	return 0;
}

802 803 804 805 806 807 808 809 810 811 812
/*
 * Module a uses b
 *  - we add 'a' as a "source", 'b' as a "target" of module use
 *  - the module_use is added to the list of 'b' sources (so
 *    'b' can walk the list to see who sourced them), and of 'a'
 *    targets (so 'a' can see what modules it targets).
 */
static int add_module_usage(struct module *a, struct module *b)
{
	struct module_use *use;

813
	pr_debug("Allocating new usage for %s.\n", a->name);
814 815
	use = kmalloc(sizeof(*use), GFP_ATOMIC);
	if (!use) {
A
Andrew Morton 已提交
816
		pr_warn("%s: out of memory loading\n", a->name);
817 818 819 820 821 822 823 824 825 826
		return -ENOMEM;
	}

	use->source = a;
	use->target = b;
	list_add(&use->source_list, &b->source_list);
	list_add(&use->target_list, &a->target_list);
	return 0;
}

827
/* Module a uses b: caller needs module_mutex() */
828
int ref_module(struct module *a, struct module *b)
L
Linus Torvalds 已提交
829
{
830
	int err;
K
Kay Sievers 已提交
831

832
	if (b == NULL || already_uses(a, b))
833 834
		return 0;

835 836
	/* If module isn't available, we fail. */
	err = strong_try_module_get(b);
837
	if (err)
838
		return err;
L
Linus Torvalds 已提交
839

840 841
	err = add_module_usage(a, b);
	if (err) {
L
Linus Torvalds 已提交
842
		module_put(b);
843
		return err;
L
Linus Torvalds 已提交
844
	}
845
	return 0;
L
Linus Torvalds 已提交
846
}
847
EXPORT_SYMBOL_GPL(ref_module);
L
Linus Torvalds 已提交
848 849 850 851

/* Clear the unload stuff of the module. */
static void module_unload_free(struct module *mod)
{
852
	struct module_use *use, *tmp;
L
Linus Torvalds 已提交
853

854
	mutex_lock(&module_mutex);
855 856
	list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
		struct module *i = use->target;
857
		pr_debug("%s unusing %s\n", mod->name, i->name);
858 859 860 861
		module_put(i);
		list_del(&use->source_list);
		list_del(&use->target_list);
		kfree(use);
L
Linus Torvalds 已提交
862
	}
863
	mutex_unlock(&module_mutex);
L
Linus Torvalds 已提交
864 865 866
}

#ifdef CONFIG_MODULE_FORCE_UNLOAD
867
static inline int try_force_unload(unsigned int flags)
L
Linus Torvalds 已提交
868 869 870
{
	int ret = (flags & O_TRUNC);
	if (ret)
871
		add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
872 873 874
	return ret;
}
#else
875
static inline int try_force_unload(unsigned int flags)
L
Linus Torvalds 已提交
876 877 878 879 880
{
	return 0;
}
#endif /* CONFIG_MODULE_FORCE_UNLOAD */

881 882
/* Try to release refcount of module, 0 means success. */
static int try_release_module_ref(struct module *mod)
L
Linus Torvalds 已提交
883
{
884
	int ret;
L
Linus Torvalds 已提交
885

886 887 888 889 890 891
	/* Try to decrement refcnt which we set at loading */
	ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
	BUG_ON(ret < 0);
	if (ret)
		/* Someone can put this right now, recover with checking */
		ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
L
Linus Torvalds 已提交
892

893 894
	return ret;
}
L
Linus Torvalds 已提交
895

896 897
static int try_stop_module(struct module *mod, int flags, int *forced)
{
898
	/* If it's not unused, quit unless we're forcing. */
899 900 901
	if (try_release_module_ref(mod) != 0) {
		*forced = try_force_unload(flags);
		if (!(*forced))
L
Linus Torvalds 已提交
902 903 904 905
			return -EWOULDBLOCK;
	}

	/* Mark it as dying. */
906
	mod->state = MODULE_STATE_GOING;
L
Linus Torvalds 已提交
907

908
	return 0;
L
Linus Torvalds 已提交
909 910
}

911 912 913 914 915 916 917 918 919 920
/**
 * module_refcount - return the refcount or -1 if unloading
 *
 * @mod:	the module we're checking
 *
 * Returns:
 *	-1 if the module is in the process of unloading
 *	otherwise the number of references in the kernel to the module
 */
int module_refcount(struct module *mod)
L
Linus Torvalds 已提交
921
{
922
	return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
L
Linus Torvalds 已提交
923 924 925 926 927 928
}
EXPORT_SYMBOL(module_refcount);

/* This exists whether we can unload or not */
static void free_module(struct module *mod);

929 930
SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
		unsigned int, flags)
L
Linus Torvalds 已提交
931 932
{
	struct module *mod;
933
	char name[MODULE_NAME_LEN];
L
Linus Torvalds 已提交
934 935
	int ret, forced = 0;

936
	if (!capable(CAP_SYS_MODULE) || modules_disabled)
937 938 939 940 941 942
		return -EPERM;

	if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
		return -EFAULT;
	name[MODULE_NAME_LEN-1] = '\0';

943 944
	if (mutex_lock_interruptible(&module_mutex) != 0)
		return -EINTR;
L
Linus Torvalds 已提交
945 946 947 948 949 950 951

	mod = find_module(name);
	if (!mod) {
		ret = -ENOENT;
		goto out;
	}

952
	if (!list_empty(&mod->source_list)) {
L
Linus Torvalds 已提交
953 954 955 956 957 958 959
		/* Other modules depend on us: get rid of them first. */
		ret = -EWOULDBLOCK;
		goto out;
	}

	/* Doing init or already dying? */
	if (mod->state != MODULE_STATE_LIVE) {
960
		/* FIXME: if (force), slam module count damn the torpedoes */
961
		pr_debug("%s already dying\n", mod->name);
L
Linus Torvalds 已提交
962 963 964 965 966
		ret = -EBUSY;
		goto out;
	}

	/* If it has an init func, it must have an exit func to unload */
R
Rusty Russell 已提交
967
	if (mod->init && !mod->exit) {
968
		forced = try_force_unload(flags);
L
Linus Torvalds 已提交
969 970 971 972 973 974 975 976 977 978 979 980
		if (!forced) {
			/* This module can't be removed */
			ret = -EBUSY;
			goto out;
		}
	}

	/* Stop the machine so refcounts can't move and disable module. */
	ret = try_stop_module(mod, flags, &forced);
	if (ret != 0)
		goto out;

981
	mutex_unlock(&module_mutex);
L
Lucas De Marchi 已提交
982
	/* Final destruction now no one is using it. */
983
	if (mod->exit != NULL)
L
Linus Torvalds 已提交
984
		mod->exit();
985 986
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
987 988
	ftrace_release_mod(mod);

989
	async_synchronize_full();
990

991
	/* Store the name of the last unloaded module for diagnostic purposes */
992
	strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
L
Linus Torvalds 已提交
993

994 995 996
	free_module(mod);
	return 0;
out:
997
	mutex_unlock(&module_mutex);
L
Linus Torvalds 已提交
998 999 1000
	return ret;
}

1001
static inline void print_unload_info(struct seq_file *m, struct module *mod)
L
Linus Torvalds 已提交
1002 1003 1004 1005
{
	struct module_use *use;
	int printed_something = 0;

1006
	seq_printf(m, " %i ", module_refcount(mod));
L
Linus Torvalds 已提交
1007

1008 1009 1010 1011
	/*
	 * Always include a trailing , so userspace can differentiate
	 * between this and the old multi-field proc format.
	 */
1012
	list_for_each_entry(use, &mod->source_list, source_list) {
L
Linus Torvalds 已提交
1013
		printed_something = 1;
1014
		seq_printf(m, "%s,", use->source->name);
L
Linus Torvalds 已提交
1015 1016 1017 1018
	}

	if (mod->init != NULL && mod->exit == NULL) {
		printed_something = 1;
1019
		seq_puts(m, "[permanent],");
L
Linus Torvalds 已提交
1020 1021 1022
	}

	if (!printed_something)
1023
		seq_puts(m, "-");
L
Linus Torvalds 已提交
1024 1025 1026 1027 1028 1029
}

void __symbol_put(const char *symbol)
{
	struct module *owner;

R
Rusty Russell 已提交
1030
	preempt_disable();
1031
	if (!find_symbol(symbol, &owner, NULL, true, false))
L
Linus Torvalds 已提交
1032 1033
		BUG();
	module_put(owner);
R
Rusty Russell 已提交
1034
	preempt_enable();
L
Linus Torvalds 已提交
1035 1036 1037
}
EXPORT_SYMBOL(__symbol_put);

1038
/* Note this assumes addr is a function, which it currently always is. */
L
Linus Torvalds 已提交
1039 1040
void symbol_put_addr(void *addr)
{
1041
	struct module *modaddr;
1042
	unsigned long a = (unsigned long)dereference_function_descriptor(addr);
L
Linus Torvalds 已提交
1043

1044
	if (core_kernel_text(a))
1045
		return;
L
Linus Torvalds 已提交
1046

1047 1048 1049 1050 1051
	/*
	 * Even though we hold a reference on the module; we still need to
	 * disable preemption in order to safely traverse the data structure.
	 */
	preempt_disable();
1052
	modaddr = __module_text_address(a);
1053
	BUG_ON(!modaddr);
1054
	module_put(modaddr);
1055
	preempt_enable();
L
Linus Torvalds 已提交
1056 1057 1058 1059
}
EXPORT_SYMBOL_GPL(symbol_put_addr);

static ssize_t show_refcnt(struct module_attribute *mattr,
1060
			   struct module_kobject *mk, char *buffer)
L
Linus Torvalds 已提交
1061
{
1062
	return sprintf(buffer, "%i\n", module_refcount(mk->mod));
L
Linus Torvalds 已提交
1063 1064
}

1065 1066
static struct module_attribute modinfo_refcnt =
	__ATTR(refcnt, 0444, show_refcnt, NULL);
L
Linus Torvalds 已提交
1067

1068 1069 1070 1071
void __module_get(struct module *module)
{
	if (module) {
		preempt_disable();
1072
		atomic_inc(&module->refcnt);
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
		trace_module_get(module, _RET_IP_);
		preempt_enable();
	}
}
EXPORT_SYMBOL(__module_get);

bool try_module_get(struct module *module)
{
	bool ret = true;

	if (module) {
		preempt_disable();
1085 1086 1087
		/* Note: here, we can fail to get a reference */
		if (likely(module_is_live(module) &&
			   atomic_inc_not_zero(&module->refcnt) != 0))
1088
			trace_module_get(module, _RET_IP_);
1089
		else
1090 1091 1092 1093 1094 1095 1096 1097
			ret = false;

		preempt_enable();
	}
	return ret;
}
EXPORT_SYMBOL(try_module_get);

A
Al Viro 已提交
1098 1099
void module_put(struct module *module)
{
1100 1101
	int ret;

A
Al Viro 已提交
1102
	if (module) {
1103
		preempt_disable();
1104 1105
		ret = atomic_dec_if_positive(&module->refcnt);
		WARN_ON(ret < 0);	/* Failed to put refcount */
1106
		trace_module_put(module, _RET_IP_);
1107
		preempt_enable();
A
Al Viro 已提交
1108 1109 1110 1111
	}
}
EXPORT_SYMBOL(module_put);

L
Linus Torvalds 已提交
1112
#else /* !CONFIG_MODULE_UNLOAD */
1113
static inline void print_unload_info(struct seq_file *m, struct module *mod)
L
Linus Torvalds 已提交
1114 1115
{
	/* We don't know the usage count, or what modules are using. */
1116
	seq_puts(m, " - -");
L
Linus Torvalds 已提交
1117 1118 1119 1120 1121 1122
}

static inline void module_unload_free(struct module *mod)
{
}

1123
int ref_module(struct module *a, struct module *b)
L
Linus Torvalds 已提交
1124
{
1125
	return strong_try_module_get(b);
L
Linus Torvalds 已提交
1126
}
1127
EXPORT_SYMBOL_GPL(ref_module);
L
Linus Torvalds 已提交
1128

1129
static inline int module_unload_init(struct module *mod)
L
Linus Torvalds 已提交
1130
{
1131
	return 0;
L
Linus Torvalds 已提交
1132 1133 1134
}
#endif /* CONFIG_MODULE_UNLOAD */

1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146
static size_t module_flags_taint(struct module *mod, char *buf)
{
	size_t l = 0;

	if (mod->taints & (1 << TAINT_PROPRIETARY_MODULE))
		buf[l++] = 'P';
	if (mod->taints & (1 << TAINT_OOT_MODULE))
		buf[l++] = 'O';
	if (mod->taints & (1 << TAINT_FORCED_MODULE))
		buf[l++] = 'F';
	if (mod->taints & (1 << TAINT_CRAP))
		buf[l++] = 'C';
1147
	if (mod->taints & (1 << TAINT_UNSIGNED_MODULE))
1148
		buf[l++] = 'E';
1149 1150
	/*
	 * TAINT_FORCED_RMMOD: could be added.
1151
	 * TAINT_CPU_OUT_OF_SPEC, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't
1152 1153 1154 1155 1156
	 * apply to modules.
	 */
	return l;
}

1157
static ssize_t show_initstate(struct module_attribute *mattr,
1158
			      struct module_kobject *mk, char *buffer)
1159 1160 1161
{
	const char *state = "unknown";

1162
	switch (mk->mod->state) {
1163 1164 1165 1166 1167 1168 1169 1170 1171
	case MODULE_STATE_LIVE:
		state = "live";
		break;
	case MODULE_STATE_COMING:
		state = "coming";
		break;
	case MODULE_STATE_GOING:
		state = "going";
		break;
1172 1173
	default:
		BUG();
1174 1175 1176 1177
	}
	return sprintf(buffer, "%s\n", state);
}

1178 1179
static struct module_attribute modinfo_initstate =
	__ATTR(initstate, 0444, show_initstate, NULL);
1180

1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191
static ssize_t store_uevent(struct module_attribute *mattr,
			    struct module_kobject *mk,
			    const char *buffer, size_t count)
{
	enum kobject_action action;

	if (kobject_action_type(buffer, count, &action) == 0)
		kobject_uevent(&mk->kobj, action);
	return count;
}

1192 1193 1194 1195 1196 1197
struct module_attribute module_uevent =
	__ATTR(uevent, 0200, NULL, store_uevent);

static ssize_t show_coresize(struct module_attribute *mattr,
			     struct module_kobject *mk, char *buffer)
{
1198
	return sprintf(buffer, "%u\n", mk->mod->core_layout.size);
1199 1200 1201 1202 1203 1204 1205 1206
}

static struct module_attribute modinfo_coresize =
	__ATTR(coresize, 0444, show_coresize, NULL);

static ssize_t show_initsize(struct module_attribute *mattr,
			     struct module_kobject *mk, char *buffer)
{
1207
	return sprintf(buffer, "%u\n", mk->mod->init_layout.size);
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224
}

static struct module_attribute modinfo_initsize =
	__ATTR(initsize, 0444, show_initsize, NULL);

static ssize_t show_taint(struct module_attribute *mattr,
			  struct module_kobject *mk, char *buffer)
{
	size_t l;

	l = module_flags_taint(mk->mod, buffer);
	buffer[l++] = '\n';
	return l;
}

static struct module_attribute modinfo_taint =
	__ATTR(taint, 0444, show_taint, NULL);
1225

1226
static struct module_attribute *modinfo_attrs[] = {
1227
	&module_uevent,
1228 1229
	&modinfo_version,
	&modinfo_srcversion,
1230 1231 1232 1233
	&modinfo_initstate,
	&modinfo_coresize,
	&modinfo_initsize,
	&modinfo_taint,
1234
#ifdef CONFIG_MODULE_UNLOAD
1235
	&modinfo_refcnt,
1236 1237 1238 1239
#endif
	NULL,
};

L
Linus Torvalds 已提交
1240 1241
static const char vermagic[] = VERMAGIC_STRING;

1242
static int try_to_force_load(struct module *mod, const char *reason)
1243 1244
{
#ifdef CONFIG_MODULE_FORCE_LOAD
A
Andi Kleen 已提交
1245
	if (!test_taint(TAINT_FORCED_MODULE))
A
Andrew Morton 已提交
1246
		pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
1247
	add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
1248 1249 1250 1251 1252 1253
	return 0;
#else
	return -ENOEXEC;
#endif
}

L
Linus Torvalds 已提交
1254
#ifdef CONFIG_MODVERSIONS
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
/* If the arch applies (non-zero) relocations to kernel kcrctab, unapply it. */
static unsigned long maybe_relocated(unsigned long crc,
				     const struct module *crc_owner)
{
#ifdef ARCH_RELOCATES_KCRCTAB
	if (crc_owner == NULL)
		return crc - (unsigned long)reloc_start;
#endif
	return crc;
}

L
Linus Torvalds 已提交
1266 1267 1268
static int check_version(Elf_Shdr *sechdrs,
			 unsigned int versindex,
			 const char *symname,
1269
			 struct module *mod,
1270 1271
			 const unsigned long *crc,
			 const struct module *crc_owner)
L
Linus Torvalds 已提交
1272 1273 1274 1275 1276 1277 1278 1279
{
	unsigned int i, num_versions;
	struct modversion_info *versions;

	/* Exporting module didn't supply crcs?  OK, we're already tainted. */
	if (!crc)
		return 1;

1280 1281 1282 1283
	/* No versions at all?  modprobe --force does this. */
	if (versindex == 0)
		return try_to_force_load(mod, symname) == 0;

L
Linus Torvalds 已提交
1284 1285 1286 1287 1288 1289 1290 1291
	versions = (void *) sechdrs[versindex].sh_addr;
	num_versions = sechdrs[versindex].sh_size
		/ sizeof(struct modversion_info);

	for (i = 0; i < num_versions; i++) {
		if (strcmp(versions[i].name, symname) != 0)
			continue;

1292
		if (versions[i].crc == maybe_relocated(*crc, crc_owner))
L
Linus Torvalds 已提交
1293
			return 1;
1294
		pr_debug("Found checksum %lX vs module %lX\n",
1295
		       maybe_relocated(*crc, crc_owner), versions[i].crc);
1296
		goto bad_version;
L
Linus Torvalds 已提交
1297
	}
1298

A
Andrew Morton 已提交
1299
	pr_warn("%s: no symbol version for %s\n", mod->name, symname);
1300
	return 0;
1301 1302

bad_version:
1303
	pr_warn("%s: disagrees about version of symbol %s\n",
1304 1305
	       mod->name, symname);
	return 0;
L
Linus Torvalds 已提交
1306 1307 1308 1309 1310 1311 1312 1313
}

static inline int check_modstruct_version(Elf_Shdr *sechdrs,
					  unsigned int versindex,
					  struct module *mod)
{
	const unsigned long *crc;

1314 1315 1316 1317 1318
	/*
	 * Since this should be found in kernel (which can't be removed), no
	 * locking is necessary -- use preempt_disable() to placate lockdep.
	 */
	preempt_disable();
R
Rusty Russell 已提交
1319
	if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL,
1320 1321
			 &crc, true, false)) {
		preempt_enable();
L
Linus Torvalds 已提交
1322
		BUG();
1323 1324
	}
	preempt_enable();
1325 1326
	return check_version(sechdrs, versindex,
			     VMLINUX_SYMBOL_STR(module_layout), mod, crc,
1327
			     NULL);
L
Linus Torvalds 已提交
1328 1329
}

1330 1331 1332
/* First part is kernel version, which we ignore if module has crcs. */
static inline int same_magic(const char *amagic, const char *bmagic,
			     bool has_crcs)
L
Linus Torvalds 已提交
1333
{
1334 1335 1336 1337
	if (has_crcs) {
		amagic += strcspn(amagic, " ");
		bmagic += strcspn(bmagic, " ");
	}
L
Linus Torvalds 已提交
1338 1339 1340 1341 1342 1343
	return strcmp(amagic, bmagic) == 0;
}
#else
static inline int check_version(Elf_Shdr *sechdrs,
				unsigned int versindex,
				const char *symname,
1344
				struct module *mod,
1345 1346
				const unsigned long *crc,
				const struct module *crc_owner)
L
Linus Torvalds 已提交
1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
{
	return 1;
}

static inline int check_modstruct_version(Elf_Shdr *sechdrs,
					  unsigned int versindex,
					  struct module *mod)
{
	return 1;
}

1358 1359
static inline int same_magic(const char *amagic, const char *bmagic,
			     bool has_crcs)
L
Linus Torvalds 已提交
1360 1361 1362 1363 1364
{
	return strcmp(amagic, bmagic) == 0;
}
#endif /* CONFIG_MODVERSIONS */

1365
/* Resolve a symbol for this module.  I.e. if we find one, record usage. */
1366 1367
static const struct kernel_symbol *resolve_symbol(struct module *mod,
						  const struct load_info *info,
1368
						  const char *name,
1369
						  char ownername[])
L
Linus Torvalds 已提交
1370 1371
{
	struct module *owner;
1372
	const struct kernel_symbol *sym;
L
Linus Torvalds 已提交
1373
	const unsigned long *crc;
1374
	int err;
L
Linus Torvalds 已提交
1375

1376 1377 1378 1379 1380 1381
	/*
	 * The module_mutex should not be a heavily contended lock;
	 * if we get the occasional sleep here, we'll go an extra iteration
	 * in the wait_event_interruptible(), which is harmless.
	 */
	sched_annotate_sleep();
1382
	mutex_lock(&module_mutex);
1383
	sym = find_symbol(name, &owner, &crc,
A
Andi Kleen 已提交
1384
			  !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
1385 1386 1387
	if (!sym)
		goto unlock;

1388 1389
	if (!check_version(info->sechdrs, info->index.vers, name, mod, crc,
			   owner)) {
1390 1391
		sym = ERR_PTR(-EINVAL);
		goto getname;
L
Linus Torvalds 已提交
1392
	}
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403

	err = ref_module(mod, owner);
	if (err) {
		sym = ERR_PTR(err);
		goto getname;
	}

getname:
	/* We must make copy under the lock if we failed to get ref. */
	strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
unlock:
1404
	mutex_unlock(&module_mutex);
1405
	return sym;
L
Linus Torvalds 已提交
1406 1407
}

1408 1409 1410 1411
static const struct kernel_symbol *
resolve_symbol_wait(struct module *mod,
		    const struct load_info *info,
		    const char *name)
1412 1413
{
	const struct kernel_symbol *ksym;
1414
	char owner[MODULE_NAME_LEN];
1415 1416

	if (wait_event_interruptible_timeout(module_wq,
1417 1418
			!IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
			|| PTR_ERR(ksym) != -EBUSY,
1419
					     30 * HZ) <= 0) {
A
Andrew Morton 已提交
1420 1421
		pr_warn("%s: gave up waiting for init of module %s.\n",
			mod->name, owner);
1422 1423 1424 1425
	}
	return ksym;
}

L
Linus Torvalds 已提交
1426 1427 1428 1429
/*
 * /sys/module/foo/sections stuff
 * J. Corbet <corbet@lwn.net>
 */
R
Rusty Russell 已提交
1430
#ifdef CONFIG_SYSFS
1431

R
Rusty Russell 已提交
1432
#ifdef CONFIG_KALLSYMS
1433 1434 1435 1436 1437
static inline bool sect_empty(const Elf_Shdr *sect)
{
	return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
}

1438
struct module_sect_attr {
1439 1440 1441 1442 1443
	struct module_attribute mattr;
	char *name;
	unsigned long address;
};

1444
struct module_sect_attrs {
1445 1446 1447 1448 1449
	struct attribute_group grp;
	unsigned int nsections;
	struct module_sect_attr attrs[0];
};

L
Linus Torvalds 已提交
1450
static ssize_t module_sect_show(struct module_attribute *mattr,
1451
				struct module_kobject *mk, char *buf)
L
Linus Torvalds 已提交
1452 1453 1454
{
	struct module_sect_attr *sattr =
		container_of(mattr, struct module_sect_attr, mattr);
1455
	return sprintf(buf, "0x%pK\n", (void *)sattr->address);
L
Linus Torvalds 已提交
1456 1457
}

1458 1459
static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
{
1460
	unsigned int section;
1461 1462 1463 1464 1465 1466

	for (section = 0; section < sect_attrs->nsections; section++)
		kfree(sect_attrs->attrs[section].name);
	kfree(sect_attrs);
}

R
Rusty Russell 已提交
1467
static void add_sect_attrs(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
1468 1469 1470 1471 1472
{
	unsigned int nloaded = 0, i, size[2];
	struct module_sect_attrs *sect_attrs;
	struct module_sect_attr *sattr;
	struct attribute **gattr;
D
Daniel Walker 已提交
1473

L
Linus Torvalds 已提交
1474
	/* Count loaded sections and allocate structures */
R
Rusty Russell 已提交
1475 1476
	for (i = 0; i < info->hdr->e_shnum; i++)
		if (!sect_empty(&info->sechdrs[i]))
L
Linus Torvalds 已提交
1477 1478 1479 1480 1481
			nloaded++;
	size[0] = ALIGN(sizeof(*sect_attrs)
			+ nloaded * sizeof(sect_attrs->attrs[0]),
			sizeof(sect_attrs->grp.attrs[0]));
	size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.attrs[0]);
1482 1483
	sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL);
	if (sect_attrs == NULL)
L
Linus Torvalds 已提交
1484 1485 1486 1487 1488 1489
		return;

	/* Setup section attributes. */
	sect_attrs->grp.name = "sections";
	sect_attrs->grp.attrs = (void *)sect_attrs + size[0];

1490
	sect_attrs->nsections = 0;
L
Linus Torvalds 已提交
1491 1492
	sattr = &sect_attrs->attrs[0];
	gattr = &sect_attrs->grp.attrs[0];
R
Rusty Russell 已提交
1493 1494 1495
	for (i = 0; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *sec = &info->sechdrs[i];
		if (sect_empty(sec))
1496
			continue;
R
Rusty Russell 已提交
1497 1498
		sattr->address = sec->sh_addr;
		sattr->name = kstrdup(info->secstrings + sec->sh_name,
1499 1500 1501 1502
					GFP_KERNEL);
		if (sattr->name == NULL)
			goto out;
		sect_attrs->nsections++;
1503
		sysfs_attr_init(&sattr->mattr.attr);
L
Linus Torvalds 已提交
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
		sattr->mattr.show = module_sect_show;
		sattr->mattr.store = NULL;
		sattr->mattr.attr.name = sattr->name;
		sattr->mattr.attr.mode = S_IRUGO;
		*(gattr++) = &(sattr++)->mattr.attr;
	}
	*gattr = NULL;

	if (sysfs_create_group(&mod->mkobj.kobj, &sect_attrs->grp))
		goto out;

	mod->sect_attrs = sect_attrs;
	return;
  out:
1518
	free_sect_attrs(sect_attrs);
L
Linus Torvalds 已提交
1519 1520 1521 1522 1523 1524 1525 1526 1527
}

static void remove_sect_attrs(struct module *mod)
{
	if (mod->sect_attrs) {
		sysfs_remove_group(&mod->mkobj.kobj,
				   &mod->sect_attrs->grp);
		/* We are positive that no one is using any sect attrs
		 * at this point.  Deallocate immediately. */
1528
		free_sect_attrs(mod->sect_attrs);
L
Linus Torvalds 已提交
1529 1530 1531 1532
		mod->sect_attrs = NULL;
	}
}

R
Roland McGrath 已提交
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542
/*
 * /sys/module/foo/notes/.section.name gives contents of SHT_NOTE sections.
 */

struct module_notes_attrs {
	struct kobject *dir;
	unsigned int notes;
	struct bin_attribute attrs[0];
};

1543
static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
R
Roland McGrath 已提交
1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560
				 struct bin_attribute *bin_attr,
				 char *buf, loff_t pos, size_t count)
{
	/*
	 * The caller checked the pos and count against our size.
	 */
	memcpy(buf, bin_attr->private + pos, count);
	return count;
}

static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
			     unsigned int i)
{
	if (notes_attrs->dir) {
		while (i-- > 0)
			sysfs_remove_bin_file(notes_attrs->dir,
					      &notes_attrs->attrs[i]);
1561
		kobject_put(notes_attrs->dir);
R
Roland McGrath 已提交
1562 1563 1564 1565
	}
	kfree(notes_attrs);
}

R
Rusty Russell 已提交
1566
static void add_notes_attrs(struct module *mod, const struct load_info *info)
R
Roland McGrath 已提交
1567 1568 1569 1570 1571
{
	unsigned int notes, loaded, i;
	struct module_notes_attrs *notes_attrs;
	struct bin_attribute *nattr;

1572 1573 1574 1575
	/* failed to create section attributes, so can't create notes */
	if (!mod->sect_attrs)
		return;

R
Roland McGrath 已提交
1576 1577
	/* Count notes sections and allocate structures.  */
	notes = 0;
R
Rusty Russell 已提交
1578 1579 1580
	for (i = 0; i < info->hdr->e_shnum; i++)
		if (!sect_empty(&info->sechdrs[i]) &&
		    (info->sechdrs[i].sh_type == SHT_NOTE))
R
Roland McGrath 已提交
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
			++notes;

	if (notes == 0)
		return;

	notes_attrs = kzalloc(sizeof(*notes_attrs)
			      + notes * sizeof(notes_attrs->attrs[0]),
			      GFP_KERNEL);
	if (notes_attrs == NULL)
		return;

	notes_attrs->notes = notes;
	nattr = &notes_attrs->attrs[0];
R
Rusty Russell 已提交
1594 1595
	for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
		if (sect_empty(&info->sechdrs[i]))
R
Roland McGrath 已提交
1596
			continue;
R
Rusty Russell 已提交
1597
		if (info->sechdrs[i].sh_type == SHT_NOTE) {
1598
			sysfs_bin_attr_init(nattr);
R
Roland McGrath 已提交
1599 1600
			nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
			nattr->attr.mode = S_IRUGO;
R
Rusty Russell 已提交
1601 1602
			nattr->size = info->sechdrs[i].sh_size;
			nattr->private = (void *) info->sechdrs[i].sh_addr;
R
Roland McGrath 已提交
1603 1604 1605 1606 1607 1608
			nattr->read = module_notes_read;
			++nattr;
		}
		++loaded;
	}

1609
	notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
R
Roland McGrath 已提交
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630
	if (!notes_attrs->dir)
		goto out;

	for (i = 0; i < notes; ++i)
		if (sysfs_create_bin_file(notes_attrs->dir,
					  &notes_attrs->attrs[i]))
			goto out;

	mod->notes_attrs = notes_attrs;
	return;

  out:
	free_notes_attrs(notes_attrs, i);
}

static void remove_notes_attrs(struct module *mod)
{
	if (mod->notes_attrs)
		free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
}

L
Linus Torvalds 已提交
1631
#else
1632

R
Rusty Russell 已提交
1633 1634
static inline void add_sect_attrs(struct module *mod,
				  const struct load_info *info)
L
Linus Torvalds 已提交
1635 1636 1637 1638 1639 1640
{
}

static inline void remove_sect_attrs(struct module *mod)
{
}
R
Roland McGrath 已提交
1641

R
Rusty Russell 已提交
1642 1643
static inline void add_notes_attrs(struct module *mod,
				   const struct load_info *info)
R
Roland McGrath 已提交
1644 1645 1646 1647 1648 1649
{
}

static inline void remove_notes_attrs(struct module *mod)
{
}
R
Rusty Russell 已提交
1650
#endif /* CONFIG_KALLSYMS */
L
Linus Torvalds 已提交
1651

1652 1653 1654 1655 1656 1657
static void add_usage_links(struct module *mod)
{
#ifdef CONFIG_MODULE_UNLOAD
	struct module_use *use;
	int nowarn;

1658
	mutex_lock(&module_mutex);
1659 1660 1661 1662
	list_for_each_entry(use, &mod->target_list, target_list) {
		nowarn = sysfs_create_link(use->target->holders_dir,
					   &mod->mkobj.kobj, mod->name);
	}
1663
	mutex_unlock(&module_mutex);
1664 1665 1666 1667 1668 1669 1670 1671
#endif
}

static void del_usage_links(struct module *mod)
{
#ifdef CONFIG_MODULE_UNLOAD
	struct module_use *use;

1672
	mutex_lock(&module_mutex);
1673 1674
	list_for_each_entry(use, &mod->target_list, target_list)
		sysfs_remove_link(use->target->holders_dir, mod->name);
1675
	mutex_unlock(&module_mutex);
1676 1677 1678
#endif
}

1679
static int module_add_modinfo_attrs(struct module *mod)
1680 1681
{
	struct module_attribute *attr;
1682
	struct module_attribute *temp_attr;
1683 1684 1685
	int error = 0;
	int i;

1686 1687 1688 1689 1690 1691 1692
	mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
					(ARRAY_SIZE(modinfo_attrs) + 1)),
					GFP_KERNEL);
	if (!mod->modinfo_attrs)
		return -ENOMEM;

	temp_attr = mod->modinfo_attrs;
1693 1694
	for (i = 0; (attr = modinfo_attrs[i]) && !error; i++) {
		if (!attr->test ||
1695 1696
		    (attr->test && attr->test(mod))) {
			memcpy(temp_attr, attr, sizeof(*temp_attr));
1697
			sysfs_attr_init(&temp_attr->attr);
1698 1699
			error = sysfs_create_file(&mod->mkobj.kobj,
					&temp_attr->attr);
1700 1701
			++temp_attr;
		}
1702 1703 1704 1705
	}
	return error;
}

1706
static void module_remove_modinfo_attrs(struct module *mod)
1707 1708 1709 1710
{
	struct module_attribute *attr;
	int i;

1711 1712 1713 1714
	for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
		/* pick a field to test for end of list */
		if (!attr->attr.name)
			break;
1715
		sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
1716 1717
		if (attr->free)
			attr->free(mod);
1718
	}
1719
	kfree(mod->modinfo_attrs);
1720
}
L
Linus Torvalds 已提交
1721

1722 1723 1724 1725 1726 1727 1728 1729
static void mod_kobject_put(struct module *mod)
{
	DECLARE_COMPLETION_ONSTACK(c);
	mod->mkobj.kobj_completion = &c;
	kobject_put(&mod->mkobj.kobj);
	wait_for_completion(&c);
}

1730
static int mod_sysfs_init(struct module *mod)
L
Linus Torvalds 已提交
1731 1732
{
	int err;
1733
	struct kobject *kobj;
L
Linus Torvalds 已提交
1734

1735
	if (!module_sysfs_initialized) {
A
Andrew Morton 已提交
1736
		pr_err("%s: module sysfs not initialized\n", mod->name);
1737 1738 1739
		err = -EINVAL;
		goto out;
	}
1740 1741 1742

	kobj = kset_find_obj(module_kset, mod->name);
	if (kobj) {
A
Andrew Morton 已提交
1743
		pr_err("%s: module is already loaded\n", mod->name);
1744 1745 1746 1747 1748
		kobject_put(kobj);
		err = -EINVAL;
		goto out;
	}

L
Linus Torvalds 已提交
1749
	mod->mkobj.mod = mod;
1750

1751 1752 1753 1754 1755
	memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
	mod->mkobj.kobj.kset = module_kset;
	err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
				   "%s", mod->name);
	if (err)
1756
		mod_kobject_put(mod);
K
Kay Sievers 已提交
1757

1758
	/* delay uevent until full sysfs population */
K
Kay Sievers 已提交
1759 1760 1761 1762
out:
	return err;
}

1763
static int mod_sysfs_setup(struct module *mod,
R
Rusty Russell 已提交
1764
			   const struct load_info *info,
K
Kay Sievers 已提交
1765 1766 1767 1768 1769
			   struct kernel_param *kparam,
			   unsigned int num_params)
{
	int err;

1770 1771 1772 1773
	err = mod_sysfs_init(mod);
	if (err)
		goto out;

1774
	mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
1775 1776
	if (!mod->holders_dir) {
		err = -ENOMEM;
K
Kay Sievers 已提交
1777
		goto out_unreg;
1778
	}
K
Kay Sievers 已提交
1779

L
Linus Torvalds 已提交
1780 1781
	err = module_param_sysfs_setup(mod, kparam, num_params);
	if (err)
K
Kay Sievers 已提交
1782
		goto out_unreg_holders;
L
Linus Torvalds 已提交
1783

1784 1785
	err = module_add_modinfo_attrs(mod);
	if (err)
1786
		goto out_unreg_param;
1787

1788
	add_usage_links(mod);
R
Rusty Russell 已提交
1789 1790
	add_sect_attrs(mod, info);
	add_notes_attrs(mod, info);
1791

1792
	kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
L
Linus Torvalds 已提交
1793 1794
	return 0;

1795 1796
out_unreg_param:
	module_param_sysfs_remove(mod);
K
Kay Sievers 已提交
1797
out_unreg_holders:
1798
	kobject_put(mod->holders_dir);
K
Kay Sievers 已提交
1799
out_unreg:
1800
	mod_kobject_put(mod);
1801
out:
L
Linus Torvalds 已提交
1802 1803
	return err;
}
1804 1805 1806

static void mod_sysfs_fini(struct module *mod)
{
R
Rusty Russell 已提交
1807 1808
	remove_notes_attrs(mod);
	remove_sect_attrs(mod);
1809
	mod_kobject_put(mod);
1810 1811
}

1812 1813 1814 1815
static void init_param_lock(struct module *mod)
{
	mutex_init(&mod->param_lock);
}
R
Rusty Russell 已提交
1816
#else /* !CONFIG_SYSFS */
1817

R
Rusty Russell 已提交
1818 1819
static int mod_sysfs_setup(struct module *mod,
			   const struct load_info *info,
1820 1821 1822 1823 1824 1825
			   struct kernel_param *kparam,
			   unsigned int num_params)
{
	return 0;
}

1826 1827 1828 1829
static void mod_sysfs_fini(struct module *mod)
{
}

1830 1831 1832 1833
static void module_remove_modinfo_attrs(struct module *mod)
{
}

1834 1835 1836 1837
static void del_usage_links(struct module *mod)
{
}

1838 1839 1840
static void init_param_lock(struct module *mod)
{
}
1841
#endif /* CONFIG_SYSFS */
L
Linus Torvalds 已提交
1842

1843
static void mod_sysfs_teardown(struct module *mod)
L
Linus Torvalds 已提交
1844
{
1845
	del_usage_links(mod);
1846
	module_remove_modinfo_attrs(mod);
L
Linus Torvalds 已提交
1847
	module_param_sysfs_remove(mod);
1848 1849
	kobject_put(mod->mkobj.drivers_dir);
	kobject_put(mod->holders_dir);
1850
	mod_sysfs_fini(mod);
L
Linus Torvalds 已提交
1851 1852
}

1853 1854 1855 1856
#ifdef CONFIG_DEBUG_SET_MODULE_RONX
/*
 * LKM RO/NX protection: protect module's text/ro-data
 * from modification and any data from execution.
R
Rusty Russell 已提交
1857 1858 1859 1860 1861 1862 1863 1864
 *
 * General layout of module is:
 *          [text] [read-only-data] [writable data]
 * text_size -----^                ^               ^
 * ro_size ------------------------|               |
 * size -------------------------------------------|
 *
 * These values are always page-aligned (as is base)
1865
 */
R
Rusty Russell 已提交
1866 1867
static void frob_text(const struct module_layout *layout,
		      int (*set_memory)(unsigned long start, int num_pages))
1868
{
R
Rusty Russell 已提交
1869 1870 1871 1872
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base,
		   layout->text_size >> PAGE_SHIFT);
1873 1874
}

R
Rusty Russell 已提交
1875 1876
static void frob_rodata(const struct module_layout *layout,
			int (*set_memory)(unsigned long start, int num_pages))
1877
{
R
Rusty Russell 已提交
1878 1879 1880 1881 1882
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->text_size & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base + layout->text_size,
		   (layout->ro_size - layout->text_size) >> PAGE_SHIFT);
1883 1884
}

R
Rusty Russell 已提交
1885 1886
static void frob_writable_data(const struct module_layout *layout,
			       int (*set_memory)(unsigned long start, int num_pages))
1887
{
R
Rusty Russell 已提交
1888 1889 1890 1891 1892
	BUG_ON((unsigned long)layout->base & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->ro_size & (PAGE_SIZE-1));
	BUG_ON((unsigned long)layout->size & (PAGE_SIZE-1));
	set_memory((unsigned long)layout->base + layout->ro_size,
		   (layout->size - layout->ro_size) >> PAGE_SHIFT);
1893 1894
}

R
Rusty Russell 已提交
1895 1896
/* livepatching wants to disable read-only so it can frob module. */
void module_disable_ro(const struct module *mod)
1897
{
R
Rusty Russell 已提交
1898 1899 1900 1901
	frob_text(&mod->core_layout, set_memory_rw);
	frob_rodata(&mod->core_layout, set_memory_rw);
	frob_text(&mod->init_layout, set_memory_rw);
	frob_rodata(&mod->init_layout, set_memory_rw);
1902
}
1903

R
Rusty Russell 已提交
1904
void module_enable_ro(const struct module *mod)
1905
{
R
Rusty Russell 已提交
1906 1907 1908 1909
	frob_text(&mod->core_layout, set_memory_ro);
	frob_rodata(&mod->core_layout, set_memory_ro);
	frob_text(&mod->init_layout, set_memory_ro);
	frob_rodata(&mod->init_layout, set_memory_ro);
1910 1911
}

R
Rusty Russell 已提交
1912
static void module_enable_nx(const struct module *mod)
1913
{
R
Rusty Russell 已提交
1914 1915 1916 1917
	frob_rodata(&mod->core_layout, set_memory_nx);
	frob_writable_data(&mod->core_layout, set_memory_nx);
	frob_rodata(&mod->init_layout, set_memory_nx);
	frob_writable_data(&mod->init_layout, set_memory_nx);
1918 1919
}

R
Rusty Russell 已提交
1920
static void module_disable_nx(const struct module *mod)
1921
{
R
Rusty Russell 已提交
1922 1923 1924 1925
	frob_rodata(&mod->core_layout, set_memory_x);
	frob_writable_data(&mod->core_layout, set_memory_x);
	frob_rodata(&mod->init_layout, set_memory_x);
	frob_writable_data(&mod->init_layout, set_memory_x);
1926 1927 1928
}

/* Iterate through all modules and set each module's text as RW */
1929
void set_all_modules_text_rw(void)
1930 1931 1932 1933 1934
{
	struct module *mod;

	mutex_lock(&module_mutex);
	list_for_each_entry_rcu(mod, &modules, list) {
1935 1936
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
R
Rusty Russell 已提交
1937 1938 1939

		frob_text(&mod->core_layout, set_memory_rw);
		frob_text(&mod->init_layout, set_memory_rw);
1940 1941 1942 1943 1944
	}
	mutex_unlock(&module_mutex);
}

/* Iterate through all modules and set each module's text as RO */
1945
void set_all_modules_text_ro(void)
1946 1947 1948 1949 1950
{
	struct module *mod;

	mutex_lock(&module_mutex);
	list_for_each_entry_rcu(mod, &modules, list) {
1951 1952
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
R
Rusty Russell 已提交
1953 1954 1955

		frob_text(&mod->core_layout, set_memory_ro);
		frob_text(&mod->init_layout, set_memory_ro);
1956 1957 1958
	}
	mutex_unlock(&module_mutex);
}
R
Rusty Russell 已提交
1959 1960 1961 1962 1963 1964 1965 1966 1967

static void disable_ro_nx(const struct module_layout *layout)
{
	frob_text(layout, set_memory_rw);
	frob_rodata(layout, set_memory_rw);
	frob_rodata(layout, set_memory_x);
	frob_writable_data(layout, set_memory_x);
}

1968
#else
R
Rusty Russell 已提交
1969 1970 1971
static void disable_ro_nx(const struct module_layout *layout) { }
static void module_enable_nx(const struct module *mod) { }
static void module_disable_nx(const struct module *mod) { }
1972 1973
#endif

1974
void __weak module_memfree(void *module_region)
1975 1976 1977 1978 1979 1980 1981 1982
{
	vfree(module_region);
}

void __weak module_arch_cleanup(struct module *mod)
{
}

1983 1984 1985 1986
void __weak module_arch_freeing_init(struct module *mod)
{
}

1987
/* Free a module, remove from lists, etc. */
L
Linus Torvalds 已提交
1988 1989
static void free_module(struct module *mod)
{
1990 1991
	trace_module_free(mod);

1992
	mod_sysfs_teardown(mod);
L
Linus Torvalds 已提交
1993

1994 1995
	/* We leave it in list to prevent duplicate loads, but make sure
	 * that noone uses it while it's being deconstructed. */
1996
	mutex_lock(&module_mutex);
1997
	mod->state = MODULE_STATE_UNFORMED;
1998
	mutex_unlock(&module_mutex);
1999

2000 2001 2002
	/* Remove dynamic debug info */
	ddebug_remove_module(mod->name);

L
Linus Torvalds 已提交
2003 2004 2005 2006 2007 2008
	/* Arch-specific cleanup. */
	module_arch_cleanup(mod);

	/* Module unload stuff */
	module_unload_free(mod);

2009 2010 2011
	/* Free any allocated parameters. */
	destroy_params(mod->kp, mod->num_kp);

2012 2013
	/* Now we can delete it from the lists */
	mutex_lock(&module_mutex);
2014 2015
	/* Unlink carefully: kallsyms could be walking list. */
	list_del_rcu(&mod->list);
2016
	mod_tree_remove(mod);
2017
	/* Remove this module from bug list, this uses list_del_rcu */
2018
	module_bug_cleanup(mod);
2019 2020
	/* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
	synchronize_sched();
2021 2022
	mutex_unlock(&module_mutex);

R
Rusty Russell 已提交
2023 2024
	/* This may be empty, but that's OK */
	disable_ro_nx(&mod->init_layout);
2025
	module_arch_freeing_init(mod);
2026
	module_memfree(mod->init_layout.base);
L
Linus Torvalds 已提交
2027
	kfree(mod->args);
2028
	percpu_modfree(mod);
2029

2030
	/* Free lock-classes; relies on the preceding sync_rcu(). */
2031
	lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
I
Ingo Molnar 已提交
2032

L
Linus Torvalds 已提交
2033
	/* Finally, free the core (containing the module structure) */
R
Rusty Russell 已提交
2034
	disable_ro_nx(&mod->core_layout);
2035
	module_memfree(mod->core_layout.base);
2036 2037 2038 2039

#ifdef CONFIG_MPU
	update_protections(current->mm);
#endif
L
Linus Torvalds 已提交
2040 2041 2042 2043 2044
}

void *__symbol_get(const char *symbol)
{
	struct module *owner;
2045
	const struct kernel_symbol *sym;
L
Linus Torvalds 已提交
2046

R
Rusty Russell 已提交
2047
	preempt_disable();
2048 2049 2050
	sym = find_symbol(symbol, &owner, NULL, true, true);
	if (sym && strong_try_module_get(owner))
		sym = NULL;
R
Rusty Russell 已提交
2051
	preempt_enable();
L
Linus Torvalds 已提交
2052

2053
	return sym ? (void *)sym->value : NULL;
L
Linus Torvalds 已提交
2054 2055 2056
}
EXPORT_SYMBOL_GPL(__symbol_get);

2057 2058
/*
 * Ensure that an exported symbol [global namespace] does not already exist
2059
 * in the kernel or in some other module's exported symbol table.
2060 2061
 *
 * You must hold the module_mutex.
2062 2063 2064
 */
static int verify_export_symbols(struct module *mod)
{
2065
	unsigned int i;
2066
	struct module *owner;
2067 2068 2069 2070 2071 2072 2073 2074
	const struct kernel_symbol *s;
	struct {
		const struct kernel_symbol *sym;
		unsigned int num;
	} arr[] = {
		{ mod->syms, mod->num_syms },
		{ mod->gpl_syms, mod->num_gpl_syms },
		{ mod->gpl_future_syms, mod->num_gpl_future_syms },
2075
#ifdef CONFIG_UNUSED_SYMBOLS
2076 2077
		{ mod->unused_syms, mod->num_unused_syms },
		{ mod->unused_gpl_syms, mod->num_unused_gpl_syms },
2078
#endif
2079
	};
2080

2081 2082
	for (i = 0; i < ARRAY_SIZE(arr); i++) {
		for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
2083
			if (find_symbol(s->name, &owner, NULL, true, false)) {
A
Andrew Morton 已提交
2084
				pr_err("%s: exports duplicate symbol %s"
2085 2086 2087 2088
				       " (owned by %s)\n",
				       mod->name, s->name, module_name(owner));
				return -ENOEXEC;
			}
2089
		}
2090 2091
	}
	return 0;
2092 2093
}

2094
/* Change all symbols so that st_value encodes the pointer directly. */
2095 2096 2097 2098
static int simplify_symbols(struct module *mod, const struct load_info *info)
{
	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
	Elf_Sym *sym = (void *)symsec->sh_addr;
L
Linus Torvalds 已提交
2099
	unsigned long secbase;
2100
	unsigned int i;
L
Linus Torvalds 已提交
2101
	int ret = 0;
2102
	const struct kernel_symbol *ksym;
L
Linus Torvalds 已提交
2103

2104 2105 2106
	for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
		const char *name = info->strtab + sym[i].st_name;

L
Linus Torvalds 已提交
2107 2108
		switch (sym[i].st_shndx) {
		case SHN_COMMON:
2109 2110 2111 2112
			/* Ignore common symbols */
			if (!strncmp(name, "__gnu_lto", 9))
				break;

L
Linus Torvalds 已提交
2113 2114
			/* We compiled with -fno-common.  These are not
			   supposed to happen.  */
2115
			pr_debug("Common symbol: %s\n", name);
2116
			pr_warn("%s: please compile with -fno-common\n",
L
Linus Torvalds 已提交
2117 2118 2119 2120 2121 2122
			       mod->name);
			ret = -ENOEXEC;
			break;

		case SHN_ABS:
			/* Don't need to do anything */
2123
			pr_debug("Absolute symbol: 0x%08lx\n",
L
Linus Torvalds 已提交
2124 2125 2126 2127
			       (long)sym[i].st_value);
			break;

		case SHN_UNDEF:
2128
			ksym = resolve_symbol_wait(mod, info, name);
L
Linus Torvalds 已提交
2129
			/* Ok if resolved.  */
2130
			if (ksym && !IS_ERR(ksym)) {
2131
				sym[i].st_value = ksym->value;
L
Linus Torvalds 已提交
2132
				break;
2133 2134
			}

L
Linus Torvalds 已提交
2135
			/* Ok if weak.  */
2136
			if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
L
Linus Torvalds 已提交
2137 2138
				break;

A
Andrew Morton 已提交
2139 2140
			pr_warn("%s: Unknown symbol %s (err %li)\n",
				mod->name, name, PTR_ERR(ksym));
2141
			ret = PTR_ERR(ksym) ?: -ENOENT;
L
Linus Torvalds 已提交
2142 2143 2144 2145
			break;

		default:
			/* Divert to percpu allocation if a percpu var. */
2146
			if (sym[i].st_shndx == info->index.pcpu)
2147
				secbase = (unsigned long)mod_percpu(mod);
L
Linus Torvalds 已提交
2148
			else
2149
				secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
L
Linus Torvalds 已提交
2150 2151 2152 2153 2154 2155 2156 2157
			sym[i].st_value += secbase;
			break;
		}
	}

	return ret;
}

2158
static int apply_relocations(struct module *mod, const struct load_info *info)
2159 2160 2161 2162 2163
{
	unsigned int i;
	int err = 0;

	/* Now do relocations. */
2164 2165
	for (i = 1; i < info->hdr->e_shnum; i++) {
		unsigned int infosec = info->sechdrs[i].sh_info;
2166 2167

		/* Not a valid relocation section? */
2168
		if (infosec >= info->hdr->e_shnum)
2169 2170 2171
			continue;

		/* Don't bother with non-allocated sections */
2172
		if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
2173 2174
			continue;

2175 2176 2177 2178 2179 2180
		if (info->sechdrs[i].sh_type == SHT_REL)
			err = apply_relocate(info->sechdrs, info->strtab,
					     info->index.sym, i, mod);
		else if (info->sechdrs[i].sh_type == SHT_RELA)
			err = apply_relocate_add(info->sechdrs, info->strtab,
						 info->index.sym, i, mod);
2181 2182 2183 2184 2185 2186
		if (err < 0)
			break;
	}
	return err;
}

2187 2188 2189 2190 2191 2192 2193 2194
/* Additional bytes needed by arch in front of individual sections */
unsigned int __weak arch_mod_section_prepend(struct module *mod,
					     unsigned int section)
{
	/* default implementation just returns zero */
	return 0;
}

L
Linus Torvalds 已提交
2195
/* Update size with this section: return offset. */
2196 2197
static long get_offset(struct module *mod, unsigned int *size,
		       Elf_Shdr *sechdr, unsigned int section)
L
Linus Torvalds 已提交
2198 2199 2200
{
	long ret;

2201
	*size += arch_mod_section_prepend(mod, section);
L
Linus Torvalds 已提交
2202 2203 2204 2205 2206 2207 2208 2209 2210
	ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
	*size = ret + sechdr->sh_size;
	return ret;
}

/* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
   might -- code, read-only data, read-write data, small data.  Tally
   sizes, and place the offsets into sh_entsize fields: high bit means it
   belongs in init. */
2211
static void layout_sections(struct module *mod, struct load_info *info)
L
Linus Torvalds 已提交
2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
{
	static unsigned long const masks[][2] = {
		/* NOTE: all executable code must be the first section
		 * in this array; otherwise modify the text_size
		 * finder in the two loops below */
		{ SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
		{ SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
		{ SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
		{ ARCH_SHF_SMALL | SHF_ALLOC, 0 }
	};
	unsigned int m, i;

2224 2225
	for (i = 0; i < info->hdr->e_shnum; i++)
		info->sechdrs[i].sh_entsize = ~0UL;
L
Linus Torvalds 已提交
2226

2227
	pr_debug("Core section allocation order:\n");
L
Linus Torvalds 已提交
2228
	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
2229 2230 2231
		for (i = 0; i < info->hdr->e_shnum; ++i) {
			Elf_Shdr *s = &info->sechdrs[i];
			const char *sname = info->secstrings + s->sh_name;
L
Linus Torvalds 已提交
2232 2233 2234 2235

			if ((s->sh_flags & masks[m][0]) != masks[m][0]
			    || (s->sh_flags & masks[m][1])
			    || s->sh_entsize != ~0UL
2236
			    || strstarts(sname, ".init"))
L
Linus Torvalds 已提交
2237
				continue;
2238
			s->sh_entsize = get_offset(mod, &mod->core_layout.size, s, i);
2239
			pr_debug("\t%s\n", sname);
L
Linus Torvalds 已提交
2240
		}
2241 2242
		switch (m) {
		case 0: /* executable */
2243 2244
			mod->core_layout.size = debug_align(mod->core_layout.size);
			mod->core_layout.text_size = mod->core_layout.size;
2245 2246
			break;
		case 1: /* RO: text and ro-data */
2247 2248
			mod->core_layout.size = debug_align(mod->core_layout.size);
			mod->core_layout.ro_size = mod->core_layout.size;
2249 2250
			break;
		case 3: /* whole core */
2251
			mod->core_layout.size = debug_align(mod->core_layout.size);
2252 2253
			break;
		}
L
Linus Torvalds 已提交
2254 2255
	}

2256
	pr_debug("Init section allocation order:\n");
L
Linus Torvalds 已提交
2257
	for (m = 0; m < ARRAY_SIZE(masks); ++m) {
2258 2259 2260
		for (i = 0; i < info->hdr->e_shnum; ++i) {
			Elf_Shdr *s = &info->sechdrs[i];
			const char *sname = info->secstrings + s->sh_name;
L
Linus Torvalds 已提交
2261 2262 2263 2264

			if ((s->sh_flags & masks[m][0]) != masks[m][0]
			    || (s->sh_flags & masks[m][1])
			    || s->sh_entsize != ~0UL
2265
			    || !strstarts(sname, ".init"))
L
Linus Torvalds 已提交
2266
				continue;
2267
			s->sh_entsize = (get_offset(mod, &mod->init_layout.size, s, i)
L
Linus Torvalds 已提交
2268
					 | INIT_OFFSET_MASK);
2269
			pr_debug("\t%s\n", sname);
L
Linus Torvalds 已提交
2270
		}
2271 2272
		switch (m) {
		case 0: /* executable */
2273 2274
			mod->init_layout.size = debug_align(mod->init_layout.size);
			mod->init_layout.text_size = mod->init_layout.size;
2275 2276
			break;
		case 1: /* RO: text and ro-data */
2277 2278
			mod->init_layout.size = debug_align(mod->init_layout.size);
			mod->init_layout.ro_size = mod->init_layout.size;
2279 2280
			break;
		case 3: /* whole init */
2281
			mod->init_layout.size = debug_align(mod->init_layout.size);
2282 2283
			break;
		}
L
Linus Torvalds 已提交
2284 2285 2286 2287 2288 2289 2290 2291
	}
}

static void set_license(struct module *mod, const char *license)
{
	if (!license)
		license = "unspecified";

2292
	if (!license_is_gpl_compatible(license)) {
A
Andi Kleen 已提交
2293
		if (!test_taint(TAINT_PROPRIETARY_MODULE))
A
Andrew Morton 已提交
2294 2295
			pr_warn("%s: module license '%s' taints kernel.\n",
				mod->name, license);
2296 2297
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
L
Linus Torvalds 已提交
2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319
	}
}

/* Parse tag=value strings from .modinfo section */
static char *next_string(char *string, unsigned long *secsize)
{
	/* Skip non-zero chars */
	while (string[0]) {
		string++;
		if ((*secsize)-- <= 1)
			return NULL;
	}

	/* Skip any zero padding. */
	while (!string[0]) {
		string++;
		if ((*secsize)-- <= 1)
			return NULL;
	}
	return string;
}

2320
static char *get_modinfo(struct load_info *info, const char *tag)
L
Linus Torvalds 已提交
2321 2322 2323
{
	char *p;
	unsigned int taglen = strlen(tag);
2324 2325
	Elf_Shdr *infosec = &info->sechdrs[info->index.info];
	unsigned long size = infosec->sh_size;
L
Linus Torvalds 已提交
2326

2327
	for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
L
Linus Torvalds 已提交
2328 2329 2330 2331 2332 2333
		if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
			return p + taglen + 1;
	}
	return NULL;
}

2334
static void setup_modinfo(struct module *mod, struct load_info *info)
2335 2336 2337 2338 2339 2340
{
	struct module_attribute *attr;
	int i;

	for (i = 0; (attr = modinfo_attrs[i]); i++) {
		if (attr->setup)
2341
			attr->setup(mod, get_modinfo(info, attr->attr.name));
2342 2343 2344
	}
}

2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355
static void free_modinfo(struct module *mod)
{
	struct module_attribute *attr;
	int i;

	for (i = 0; (attr = modinfo_attrs[i]); i++) {
		if (attr->free)
			attr->free(mod);
	}
}

L
Linus Torvalds 已提交
2356
#ifdef CONFIG_KALLSYMS
2357 2358 2359 2360 2361 2362

/* lookup symbol in given range of kernel_symbols */
static const struct kernel_symbol *lookup_symbol(const char *name,
	const struct kernel_symbol *start,
	const struct kernel_symbol *stop)
{
2363 2364
	return bsearch(name, start, stop - start,
			sizeof(struct kernel_symbol), cmp_name);
2365 2366
}

2367 2368
static int is_exported(const char *name, unsigned long value,
		       const struct module *mod)
L
Linus Torvalds 已提交
2369
{
2370 2371 2372
	const struct kernel_symbol *ks;
	if (!mod)
		ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
2373
	else
2374 2375
		ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
	return ks != NULL && ks->value == value;
L
Linus Torvalds 已提交
2376 2377 2378
}

/* As per nm */
2379
static char elf_type(const Elf_Sym *sym, const struct load_info *info)
L
Linus Torvalds 已提交
2380
{
2381 2382
	const Elf_Shdr *sechdrs = info->sechdrs;

L
Linus Torvalds 已提交
2383 2384 2385 2386 2387 2388 2389 2390
	if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
		if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
			return 'v';
		else
			return 'w';
	}
	if (sym->st_shndx == SHN_UNDEF)
		return 'U';
2391
	if (sym->st_shndx == SHN_ABS || sym->st_shndx == info->index.pcpu)
L
Linus Torvalds 已提交
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411
		return 'a';
	if (sym->st_shndx >= SHN_LORESERVE)
		return '?';
	if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
		return 't';
	if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC
	    && sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
		if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
			return 'r';
		else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
			return 'g';
		else
			return 'd';
	}
	if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
		if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
			return 's';
		else
			return 'b';
	}
2412 2413
	if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
		      ".debug")) {
L
Linus Torvalds 已提交
2414
		return 'n';
2415
	}
L
Linus Torvalds 已提交
2416 2417 2418
	return '?';
}

2419
static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
2420
			unsigned int shnum, unsigned int pcpundx)
2421 2422 2423 2424 2425 2426 2427 2428
{
	const Elf_Shdr *sec;

	if (src->st_shndx == SHN_UNDEF
	    || src->st_shndx >= shnum
	    || !src->st_name)
		return false;

2429 2430 2431 2432 2433
#ifdef CONFIG_KALLSYMS_ALL
	if (src->st_shndx == pcpundx)
		return true;
#endif

2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444
	sec = sechdrs + src->st_shndx;
	if (!(sec->sh_flags & SHF_ALLOC)
#ifndef CONFIG_KALLSYMS_ALL
	    || !(sec->sh_flags & SHF_EXECINSTR)
#endif
	    || (sec->sh_entsize & INIT_OFFSET_MASK))
		return false;

	return true;
}

2445 2446 2447 2448 2449 2450 2451
/*
 * We only allocate and copy the strings needed by the parts of symtab
 * we keep.  This is simple, but has the effect of making multiple
 * copies of duplicates.  We could be more sophisticated, see
 * linux-kernel thread starting with
 * <73defb5e4bca04a6431392cc341112b1@localhost>.
 */
2452
static void layout_symtab(struct module *mod, struct load_info *info)
2453
{
2454 2455
	Elf_Shdr *symsect = info->sechdrs + info->index.sym;
	Elf_Shdr *strsect = info->sechdrs + info->index.str;
2456
	const Elf_Sym *src;
2457
	unsigned int i, nsrc, ndst, strtab_size = 0;
2458 2459 2460

	/* Put symbol section at end of init part of module. */
	symsect->sh_flags |= SHF_ALLOC;
2461
	symsect->sh_entsize = get_offset(mod, &mod->init_layout.size, symsect,
2462
					 info->index.sym) | INIT_OFFSET_MASK;
2463
	pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
2464

2465
	src = (void *)info->hdr + symsect->sh_offset;
2466
	nsrc = symsect->sh_size / sizeof(*src);
2467

2468
	/* Compute total space required for the core symbols' strtab. */
2469 2470
	for (ndst = i = 0; i < nsrc; i++) {
		if (i == 0 ||
2471 2472
		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
				   info->index.pcpu)) {
2473
			strtab_size += strlen(&info->strtab[src[i].st_name])+1;
2474
			ndst++;
2475
		}
2476
	}
2477 2478

	/* Append room for core symbols at end of core part. */
2479 2480 2481 2482
	info->symoffs = ALIGN(mod->core_layout.size, symsect->sh_addralign ?: 1);
	info->stroffs = mod->core_layout.size = info->symoffs + ndst * sizeof(Elf_Sym);
	mod->core_layout.size += strtab_size;
	mod->core_layout.size = debug_align(mod->core_layout.size);
2483

2484 2485
	/* Put string table section at end of init part of module. */
	strsect->sh_flags |= SHF_ALLOC;
2486
	strsect->sh_entsize = get_offset(mod, &mod->init_layout.size, strsect,
2487
					 info->index.str) | INIT_OFFSET_MASK;
2488
	pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
2489 2490 2491 2492 2493 2494 2495

	/* We'll tack temporary mod_kallsyms on the end. */
	mod->init_layout.size = ALIGN(mod->init_layout.size,
				      __alignof__(struct mod_kallsyms));
	info->mod_kallsyms_init_off = mod->init_layout.size;
	mod->init_layout.size += sizeof(struct mod_kallsyms);
	mod->init_layout.size = debug_align(mod->init_layout.size);
2496 2497
}

2498 2499 2500 2501 2502
/*
 * We use the full symtab and strtab which layout_symtab arranged to
 * be appended to the init section.  Later we switch to the cut-down
 * core-only ones.
 */
2503
static void add_kallsyms(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
2504
{
2505 2506 2507
	unsigned int i, ndst;
	const Elf_Sym *src;
	Elf_Sym *dst;
2508
	char *s;
2509
	Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
L
Linus Torvalds 已提交
2510

2511 2512 2513 2514 2515
	/* Set up to point into init section. */
	mod->kallsyms = mod->init_layout.base + info->mod_kallsyms_init_off;

	mod->kallsyms->symtab = (void *)symsec->sh_addr;
	mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
2516
	/* Make sure we get permanent strtab: don't use info->strtab. */
2517
	mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
L
Linus Torvalds 已提交
2518 2519

	/* Set types up while we still have access to sections. */
2520 2521 2522 2523 2524 2525 2526 2527 2528
	for (i = 0; i < mod->kallsyms->num_symtab; i++)
		mod->kallsyms->symtab[i].st_info
			= elf_type(&mod->kallsyms->symtab[i], info);

	/* Now populate the cut down core kallsyms for after init. */
	mod->core_kallsyms.symtab = dst = mod->core_layout.base + info->symoffs;
	mod->core_kallsyms.strtab = s = mod->core_layout.base + info->stroffs;
	src = mod->kallsyms->symtab;
	for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
2529
		if (i == 0 ||
2530 2531
		    is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum,
				   info->index.pcpu)) {
2532
			dst[ndst] = src[i];
2533 2534
			dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
			s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
2535 2536
				     KSYM_NAME_LEN) + 1;
		}
2537
	}
2538
	mod->core_kallsyms.num_symtab = ndst;
L
Linus Torvalds 已提交
2539 2540
}
#else
2541
static inline void layout_symtab(struct module *mod, struct load_info *info)
2542 2543
{
}
2544

2545
static void add_kallsyms(struct module *mod, const struct load_info *info)
L
Linus Torvalds 已提交
2546 2547 2548 2549
{
}
#endif /* CONFIG_KALLSYMS */

2550
static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
2551
{
2552 2553
	if (!debug)
		return;
2554 2555
#ifdef CONFIG_DYNAMIC_DEBUG
	if (ddebug_add_module(debug, num, debug->modname))
A
Andrew Morton 已提交
2556 2557
		pr_err("dynamic debug error adding module: %s\n",
			debug->modname);
2558
#endif
R
Rusty Russell 已提交
2559
}
2560

2561 2562 2563 2564 2565 2566
static void dynamic_debug_remove(struct _ddebug *debug)
{
	if (debug)
		ddebug_remove_module(debug->modname);
}

2567 2568
void * __weak module_alloc(unsigned long size)
{
R
Rusty Russell 已提交
2569
	return vmalloc_exec(size);
2570 2571
}

C
Catalin Marinas 已提交
2572
#ifdef CONFIG_DEBUG_KMEMLEAK
2573 2574
static void kmemleak_load_module(const struct module *mod,
				 const struct load_info *info)
C
Catalin Marinas 已提交
2575 2576 2577 2578
{
	unsigned int i;

	/* only scan the sections containing data */
2579
	kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
C
Catalin Marinas 已提交
2580

2581
	for (i = 1; i < info->hdr->e_shnum; i++) {
2582 2583 2584 2585
		/* Scan all writable sections that's not executable */
		if (!(info->sechdrs[i].sh_flags & SHF_ALLOC) ||
		    !(info->sechdrs[i].sh_flags & SHF_WRITE) ||
		    (info->sechdrs[i].sh_flags & SHF_EXECINSTR))
C
Catalin Marinas 已提交
2586 2587
			continue;

2588 2589
		kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
				   info->sechdrs[i].sh_size, GFP_KERNEL);
C
Catalin Marinas 已提交
2590 2591 2592
	}
}
#else
2593 2594
static inline void kmemleak_load_module(const struct module *mod,
					const struct load_info *info)
C
Catalin Marinas 已提交
2595 2596 2597 2598
{
}
#endif

R
Rusty Russell 已提交
2599
#ifdef CONFIG_MODULE_SIG
2600
static int module_sig_check(struct load_info *info)
R
Rusty Russell 已提交
2601 2602
{
	int err = -ENOKEY;
2603 2604
	const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
	const void *mod = info->hdr;
2605

2606 2607
	if (info->len > markerlen &&
	    memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
2608
		/* We truncate the module to discard the signature */
2609 2610
		info->len -= markerlen;
		err = mod_verify_sig(mod, &info->len);
R
Rusty Russell 已提交
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624
	}

	if (!err) {
		info->sig_ok = true;
		return 0;
	}

	/* Not having a signature is only an error if we're strict. */
	if (err == -ENOKEY && !sig_enforce)
		err = 0;

	return err;
}
#else /* !CONFIG_MODULE_SIG */
2625
static int module_sig_check(struct load_info *info)
R
Rusty Russell 已提交
2626 2627 2628 2629 2630
{
	return 0;
}
#endif /* !CONFIG_MODULE_SIG */

2631 2632
/* Sanity checks against invalid binaries, wrong arch, weird elf version. */
static int elf_header_check(struct load_info *info)
2633
{
2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646
	if (info->len < sizeof(*(info->hdr)))
		return -ENOEXEC;

	if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
	    || info->hdr->e_type != ET_REL
	    || !elf_check_arch(info->hdr)
	    || info->hdr->e_shentsize != sizeof(Elf_Shdr))
		return -ENOEXEC;

	if (info->hdr->e_shoff >= info->len
	    || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
		info->len - info->hdr->e_shoff))
		return -ENOEXEC;
2647

2648 2649 2650
	return 0;
}

2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667
#define COPY_CHUNK_SIZE (16*PAGE_SIZE)

static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
{
	do {
		unsigned long n = min(len, COPY_CHUNK_SIZE);

		if (copy_from_user(dst, usrc, n) != 0)
			return -EFAULT;
		cond_resched();
		dst += n;
		usrc += n;
		len -= n;
	} while (len);
	return 0;
}

2668 2669 2670
/* Sets info->hdr and info->len. */
static int copy_module_from_user(const void __user *umod, unsigned long len,
				  struct load_info *info)
2671 2672 2673
{
	int err;

2674 2675
	info->len = len;
	if (info->len < sizeof(*(info->hdr)))
2676 2677
		return -ENOEXEC;

2678 2679 2680 2681
	err = security_kernel_module_from_file(NULL);
	if (err)
		return err;

2682
	/* Suck in entire file: we'll want most of it. */
2683 2684
	info->hdr = __vmalloc(info->len,
			GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN, PAGE_KERNEL);
2685
	if (!info->hdr)
2686 2687
		return -ENOMEM;

2688
	if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
2689 2690
		vfree(info->hdr);
		return -EFAULT;
2691 2692
	}

2693 2694 2695 2696 2697 2698
	return 0;
}

/* Sets info->hdr and info->len. */
static int copy_module_from_fd(int fd, struct load_info *info)
{
A
Al Viro 已提交
2699
	struct fd f = fdget(fd);
2700 2701 2702 2703 2704
	int err;
	struct kstat stat;
	loff_t pos;
	ssize_t bytes = 0;

A
Al Viro 已提交
2705
	if (!f.file)
2706 2707
		return -ENOEXEC;

A
Al Viro 已提交
2708
	err = security_kernel_module_from_file(f.file);
R
Rusty Russell 已提交
2709
	if (err)
2710
		goto out;
R
Rusty Russell 已提交
2711

A
Al Viro 已提交
2712
	err = vfs_getattr(&f.file->f_path, &stat);
R
Rusty Russell 已提交
2713
	if (err)
2714
		goto out;
2715

2716 2717 2718
	if (stat.size > INT_MAX) {
		err = -EFBIG;
		goto out;
2719
	}
2720 2721 2722 2723 2724 2725 2726

	/* Don't hand 0 to vmalloc, it whines. */
	if (stat.size == 0) {
		err = -EINVAL;
		goto out;
	}

2727 2728 2729 2730
	info->hdr = vmalloc(stat.size);
	if (!info->hdr) {
		err = -ENOMEM;
		goto out;
2731
	}
R
Rusty Russell 已提交
2732

2733 2734
	pos = 0;
	while (pos < stat.size) {
A
Al Viro 已提交
2735
		bytes = kernel_read(f.file, pos, (char *)(info->hdr) + pos,
2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
				    stat.size - pos);
		if (bytes < 0) {
			vfree(info->hdr);
			err = bytes;
			goto out;
		}
		if (bytes == 0)
			break;
		pos += bytes;
	}
	info->len = pos;
2747

2748
out:
A
Al Viro 已提交
2749
	fdput(f);
2750 2751 2752
	return err;
}

R
Rusty Russell 已提交
2753 2754 2755 2756 2757
static void free_copy(struct load_info *info)
{
	vfree(info->hdr);
}

2758
static int rewrite_section_headers(struct load_info *info, int flags)
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768
{
	unsigned int i;

	/* This should always be true, but let's be sure. */
	info->sechdrs[0].sh_addr = 0;

	for (i = 1; i < info->hdr->e_shnum; i++) {
		Elf_Shdr *shdr = &info->sechdrs[i];
		if (shdr->sh_type != SHT_NOBITS
		    && info->len < shdr->sh_offset + shdr->sh_size) {
A
Andrew Morton 已提交
2769
			pr_err("Module len %lu truncated\n", info->len);
2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782
			return -ENOEXEC;
		}

		/* Mark all sections sh_addr with their address in the
		   temporary image. */
		shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;

#ifndef CONFIG_MODULE_UNLOAD
		/* Don't load .exit sections */
		if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
			shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
#endif
	}
2783 2784

	/* Track but don't keep modinfo and version sections. */
2785 2786 2787 2788
	if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
		info->index.vers = 0; /* Pretend no __versions section! */
	else
		info->index.vers = find_sec(info, "__versions");
2789
	info->index.info = find_sec(info, ".modinfo");
2790 2791
	info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
	info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
2792 2793 2794
	return 0;
}

L
Linus Torvalds 已提交
2795 2796 2797 2798 2799 2800 2801 2802
/*
 * Set up our basic convenience variables (pointers to section headers,
 * search for module section index etc), and do some basic section
 * verification.
 *
 * Return the temporary module pointer (we'll replace it with the final
 * one when we move the module sections around).
 */
2803
static struct module *setup_load_info(struct load_info *info, int flags)
L
Linus Torvalds 已提交
2804 2805
{
	unsigned int i;
2806
	int err;
L
Linus Torvalds 已提交
2807 2808 2809 2810
	struct module *mod;

	/* Set up the convenience variables */
	info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
2811 2812
	info->secstrings = (void *)info->hdr
		+ info->sechdrs[info->hdr->e_shstrndx].sh_offset;
L
Linus Torvalds 已提交
2813

2814
	err = rewrite_section_headers(info, flags);
2815 2816
	if (err)
		return ERR_PTR(err);
L
Linus Torvalds 已提交
2817

2818 2819
	/* Find internal symbols and strings. */
	for (i = 1; i < info->hdr->e_shnum; i++) {
L
Linus Torvalds 已提交
2820 2821 2822
		if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
			info->index.sym = i;
			info->index.str = info->sechdrs[i].sh_link;
2823 2824 2825
			info->strtab = (char *)info->hdr
				+ info->sechdrs[info->index.str].sh_offset;
			break;
L
Linus Torvalds 已提交
2826 2827 2828
		}
	}

2829
	info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
L
Linus Torvalds 已提交
2830
	if (!info->index.mod) {
A
Andrew Morton 已提交
2831
		pr_warn("No module found in object\n");
L
Linus Torvalds 已提交
2832 2833 2834 2835 2836 2837
		return ERR_PTR(-ENOEXEC);
	}
	/* This is temporary: point mod into copy of data. */
	mod = (void *)info->sechdrs[info->index.mod].sh_addr;

	if (info->index.sym == 0) {
A
Andrew Morton 已提交
2838
		pr_warn("%s: module has no symbols (stripped?)\n", mod->name);
L
Linus Torvalds 已提交
2839 2840 2841
		return ERR_PTR(-ENOEXEC);
	}

2842
	info->index.pcpu = find_pcpusec(info);
L
Linus Torvalds 已提交
2843 2844 2845 2846 2847 2848 2849 2850

	/* Check module struct version now, before we try to use module. */
	if (!check_modstruct_version(info->sechdrs, info->index.vers, mod))
		return ERR_PTR(-ENOEXEC);

	return mod;
}

2851
static int check_modinfo(struct module *mod, struct load_info *info, int flags)
2852
{
2853
	const char *modmagic = get_modinfo(info, "vermagic");
2854 2855
	int err;

2856 2857 2858
	if (flags & MODULE_INIT_IGNORE_VERMAGIC)
		modmagic = NULL;

2859 2860 2861 2862 2863
	/* This is allowed: modprobe --force will invalidate it. */
	if (!modmagic) {
		err = try_to_force_load(mod, "bad vermagic");
		if (err)
			return err;
2864
	} else if (!same_magic(modmagic, vermagic, info->index.vers)) {
A
Andrew Morton 已提交
2865
		pr_err("%s: version magic '%s' should be '%s'\n",
2866 2867 2868 2869
		       mod->name, modmagic, vermagic);
		return -ENOEXEC;
	}

2870
	if (!get_modinfo(info, "intree"))
2871
		add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
2872

2873
	if (get_modinfo(info, "staging")) {
2874
		add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
A
Andrew Morton 已提交
2875 2876
		pr_warn("%s: module is from the staging directory, the quality "
			"is unknown, you have been warned.\n", mod->name);
2877
	}
2878 2879

	/* Set up license info based on the info section */
2880
	set_license(mod, get_modinfo(info, "license"));
2881

2882 2883 2884
	return 0;
}

2885
static int find_module_sections(struct module *mod, struct load_info *info)
L
Linus Torvalds 已提交
2886
{
2887
	mod->kp = section_objs(info, "__param",
L
Linus Torvalds 已提交
2888
			       sizeof(*mod->kp), &mod->num_kp);
2889
	mod->syms = section_objs(info, "__ksymtab",
L
Linus Torvalds 已提交
2890
				 sizeof(*mod->syms), &mod->num_syms);
2891 2892
	mod->crcs = section_addr(info, "__kcrctab");
	mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
L
Linus Torvalds 已提交
2893 2894
				     sizeof(*mod->gpl_syms),
				     &mod->num_gpl_syms);
2895 2896
	mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
	mod->gpl_future_syms = section_objs(info,
L
Linus Torvalds 已提交
2897 2898 2899
					    "__ksymtab_gpl_future",
					    sizeof(*mod->gpl_future_syms),
					    &mod->num_gpl_future_syms);
2900
	mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
L
Linus Torvalds 已提交
2901 2902

#ifdef CONFIG_UNUSED_SYMBOLS
2903
	mod->unused_syms = section_objs(info, "__ksymtab_unused",
L
Linus Torvalds 已提交
2904 2905
					sizeof(*mod->unused_syms),
					&mod->num_unused_syms);
2906 2907
	mod->unused_crcs = section_addr(info, "__kcrctab_unused");
	mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
L
Linus Torvalds 已提交
2908 2909
					    sizeof(*mod->unused_gpl_syms),
					    &mod->num_unused_gpl_syms);
2910
	mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
L
Linus Torvalds 已提交
2911 2912
#endif
#ifdef CONFIG_CONSTRUCTORS
2913
	mod->ctors = section_objs(info, ".ctors",
L
Linus Torvalds 已提交
2914
				  sizeof(*mod->ctors), &mod->num_ctors);
2915 2916 2917 2918 2919 2920 2921 2922
	if (!mod->ctors)
		mod->ctors = section_objs(info, ".init_array",
				sizeof(*mod->ctors), &mod->num_ctors);
	else if (find_sec(info, ".init_array")) {
		/*
		 * This shouldn't happen with same compiler and binutils
		 * building all parts of the module.
		 */
2923
		pr_warn("%s: has both .ctors and .init_array.\n",
2924 2925 2926
		       mod->name);
		return -EINVAL;
	}
L
Linus Torvalds 已提交
2927 2928 2929
#endif

#ifdef CONFIG_TRACEPOINTS
2930 2931 2932
	mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
					     sizeof(*mod->tracepoints_ptrs),
					     &mod->num_tracepoints);
L
Linus Torvalds 已提交
2933
#endif
2934 2935 2936 2937 2938
#ifdef HAVE_JUMP_LABEL
	mod->jump_entries = section_objs(info, "__jump_table",
					sizeof(*mod->jump_entries),
					&mod->num_jump_entries);
#endif
L
Linus Torvalds 已提交
2939
#ifdef CONFIG_EVENT_TRACING
2940
	mod->trace_events = section_objs(info, "_ftrace_events",
L
Linus Torvalds 已提交
2941 2942
					 sizeof(*mod->trace_events),
					 &mod->num_trace_events);
2943 2944 2945
	mod->trace_enums = section_objs(info, "_ftrace_enum_map",
					sizeof(*mod->trace_enums),
					&mod->num_trace_enums);
L
Linus Torvalds 已提交
2946
#endif
2947 2948 2949 2950 2951
#ifdef CONFIG_TRACING
	mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
					 sizeof(*mod->trace_bprintk_fmt_start),
					 &mod->num_trace_bprintk_fmt);
#endif
L
Linus Torvalds 已提交
2952 2953
#ifdef CONFIG_FTRACE_MCOUNT_RECORD
	/* sechdrs[0].sh_size is always zero */
2954
	mod->ftrace_callsites = section_objs(info, "__mcount_loc",
L
Linus Torvalds 已提交
2955 2956 2957
					     sizeof(*mod->ftrace_callsites),
					     &mod->num_ftrace_callsites);
#endif
2958

2959 2960 2961
	mod->extable = section_objs(info, "__ex_table",
				    sizeof(*mod->extable), &mod->num_exentries);

2962
	if (section_addr(info, "__obsparm"))
A
Andrew Morton 已提交
2963
		pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
2964 2965 2966

	info->debug = section_objs(info, "__verbose",
				   sizeof(*info->debug), &info->num_debug);
2967 2968

	return 0;
L
Linus Torvalds 已提交
2969 2970
}

2971
static int move_module(struct module *mod, struct load_info *info)
2972 2973 2974 2975 2976
{
	int i;
	void *ptr;

	/* Do the allocs. */
2977
	ptr = module_alloc(mod->core_layout.size);
2978 2979 2980 2981 2982 2983 2984
	/*
	 * The pointer to this block is stored in the module structure
	 * which is inside the block. Just mark it as not being a
	 * leak.
	 */
	kmemleak_not_leak(ptr);
	if (!ptr)
R
Rusty Russell 已提交
2985
		return -ENOMEM;
2986

2987 2988
	memset(ptr, 0, mod->core_layout.size);
	mod->core_layout.base = ptr;
2989

2990 2991
	if (mod->init_layout.size) {
		ptr = module_alloc(mod->init_layout.size);
R
Rusty Russell 已提交
2992 2993 2994 2995 2996 2997 2998 2999
		/*
		 * The pointer to this block is stored in the module structure
		 * which is inside the block. This block doesn't need to be
		 * scanned as it contains data and code that will be freed
		 * after the module is initialized.
		 */
		kmemleak_ignore(ptr);
		if (!ptr) {
3000
			module_memfree(mod->core_layout.base);
R
Rusty Russell 已提交
3001 3002
			return -ENOMEM;
		}
3003 3004
		memset(ptr, 0, mod->init_layout.size);
		mod->init_layout.base = ptr;
R
Rusty Russell 已提交
3005
	} else
3006
		mod->init_layout.base = NULL;
3007 3008

	/* Transfer each section which specifies SHF_ALLOC */
3009
	pr_debug("final section addresses:\n");
3010
	for (i = 0; i < info->hdr->e_shnum; i++) {
3011
		void *dest;
3012
		Elf_Shdr *shdr = &info->sechdrs[i];
3013

3014
		if (!(shdr->sh_flags & SHF_ALLOC))
3015 3016
			continue;

3017
		if (shdr->sh_entsize & INIT_OFFSET_MASK)
3018
			dest = mod->init_layout.base
3019
				+ (shdr->sh_entsize & ~INIT_OFFSET_MASK);
3020
		else
3021
			dest = mod->core_layout.base + shdr->sh_entsize;
3022

3023 3024
		if (shdr->sh_type != SHT_NOBITS)
			memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
3025
		/* Update sh_addr to point to copy in image. */
3026
		shdr->sh_addr = (unsigned long)dest;
3027 3028
		pr_debug("\t0x%lx %s\n",
			 (long)shdr->sh_addr, info->secstrings + shdr->sh_name);
3029
	}
R
Rusty Russell 已提交
3030 3031

	return 0;
3032 3033
}

3034
static int check_module_license_and_versions(struct module *mod)
3035 3036 3037 3038 3039 3040 3041
{
	/*
	 * ndiswrapper is under GPL by itself, but loads proprietary modules.
	 * Don't use add_taint_module(), as it would prevent ndiswrapper from
	 * using GPL-only symbols it needs.
	 */
	if (strcmp(mod->name, "ndiswrapper") == 0)
3042
		add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
3043 3044 3045

	/* driverloader was caught wrongly pretending to be under GPL */
	if (strcmp(mod->name, "driverloader") == 0)
3046 3047
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
3048

3049 3050
	/* lve claims to be GPL but upstream won't provide source */
	if (strcmp(mod->name, "lve") == 0)
3051 3052
		add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
				 LOCKDEP_NOW_UNRELIABLE);
3053

3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082
#ifdef CONFIG_MODVERSIONS
	if ((mod->num_syms && !mod->crcs)
	    || (mod->num_gpl_syms && !mod->gpl_crcs)
	    || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
#ifdef CONFIG_UNUSED_SYMBOLS
	    || (mod->num_unused_syms && !mod->unused_crcs)
	    || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
#endif
		) {
		return try_to_force_load(mod,
					 "no versions for exported symbols");
	}
#endif
	return 0;
}

static void flush_module_icache(const struct module *mod)
{
	mm_segment_t old_fs;

	/* flush the icache in correct context */
	old_fs = get_fs();
	set_fs(KERNEL_DS);

	/*
	 * Flush the instruction cache, since we've played with text.
	 * Do it before processing of module parameters, so the module
	 * can provide parameter accessor functions of its own.
	 */
3083 3084 3085 3086 3087 3088
	if (mod->init_layout.base)
		flush_icache_range((unsigned long)mod->init_layout.base,
				   (unsigned long)mod->init_layout.base
				   + mod->init_layout.size);
	flush_icache_range((unsigned long)mod->core_layout.base,
			   (unsigned long)mod->core_layout.base + mod->core_layout.size);
3089 3090 3091 3092

	set_fs(old_fs);
}

3093 3094 3095 3096 3097 3098 3099 3100
int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
				     Elf_Shdr *sechdrs,
				     char *secstrings,
				     struct module *mod)
{
	return 0;
}

3101
static struct module *layout_and_allocate(struct load_info *info, int flags)
L
Linus Torvalds 已提交
3102
{
R
Rusty Russell 已提交
3103
	/* Module within temporary copy. */
L
Linus Torvalds 已提交
3104
	struct module *mod;
R
Rusty Russell 已提交
3105
	int err;
3106

3107
	mod = setup_load_info(info, flags);
R
Rusty Russell 已提交
3108 3109
	if (IS_ERR(mod))
		return mod;
L
Linus Torvalds 已提交
3110

3111
	err = check_modinfo(mod, info, flags);
3112 3113
	if (err)
		return ERR_PTR(err);
L
Linus Torvalds 已提交
3114 3115

	/* Allow arches to frob section contents and sizes.  */
3116 3117
	err = module_frob_arch_sections(info->hdr, info->sechdrs,
					info->secstrings, mod);
L
Linus Torvalds 已提交
3118
	if (err < 0)
3119
		return ERR_PTR(err);
L
Linus Torvalds 已提交
3120

3121 3122
	/* We will do a special allocation for per-cpu sections later. */
	info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
L
Linus Torvalds 已提交
3123 3124 3125 3126

	/* Determine total sizes, and put offsets in sh_entsize.  For now
	   this is done generically; there doesn't appear to be any
	   special cases for the architectures. */
3127 3128
	layout_sections(mod, info);
	layout_symtab(mod, info);
L
Linus Torvalds 已提交
3129

3130
	/* Allocate and move to the final place */
3131
	err = move_module(mod, info);
R
Rusty Russell 已提交
3132
	if (err)
3133
		return ERR_PTR(err);
R
Rusty Russell 已提交
3134 3135 3136

	/* Module has been copied to its final place now: return it. */
	mod = (void *)info->sechdrs[info->index.mod].sh_addr;
3137
	kmemleak_load_module(mod, info);
R
Rusty Russell 已提交
3138 3139 3140 3141 3142 3143 3144
	return mod;
}

/* mod is no longer valid after this! */
static void module_deallocate(struct module *mod, struct load_info *info)
{
	percpu_modfree(mod);
3145
	module_arch_freeing_init(mod);
3146 3147
	module_memfree(mod->init_layout.base);
	module_memfree(mod->core_layout.base);
R
Rusty Russell 已提交
3148 3149
}

3150 3151 3152 3153 3154 3155 3156
int __weak module_finalize(const Elf_Ehdr *hdr,
			   const Elf_Shdr *sechdrs,
			   struct module *me)
{
	return 0;
}

3157 3158
static int post_relocation(struct module *mod, const struct load_info *info)
{
3159
	/* Sort exception table now relocations are done. */
3160 3161 3162 3163 3164 3165
	sort_extable(mod->extable, mod->extable + mod->num_exentries);

	/* Copy relocated percpu area over. */
	percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
		       info->sechdrs[info->index.pcpu].sh_size);

3166
	/* Setup kallsyms-specific fields. */
3167 3168 3169 3170 3171 3172
	add_kallsyms(mod, info);

	/* Arch-specific module finalizing. */
	return module_finalize(info->hdr, info->sechdrs, mod);
}

3173 3174 3175 3176 3177 3178
/* Is this module of this name done loading?  No locks held. */
static bool finished_loading(const char *name)
{
	struct module *mod;
	bool ret;

3179 3180 3181 3182 3183 3184
	/*
	 * The module_mutex should not be a heavily contended lock;
	 * if we get the occasional sleep here, we'll go an extra iteration
	 * in the wait_event_interruptible(), which is harmless.
	 */
	sched_annotate_sleep();
3185
	mutex_lock(&module_mutex);
3186
	mod = find_module_all(name, strlen(name), true);
3187 3188
	ret = !mod || mod->state == MODULE_STATE_LIVE
		|| mod->state == MODULE_STATE_GOING;
3189 3190 3191 3192 3193
	mutex_unlock(&module_mutex);

	return ret;
}

3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204
/* Call module constructors. */
static void do_mod_ctors(struct module *mod)
{
#ifdef CONFIG_CONSTRUCTORS
	unsigned long i;

	for (i = 0; i < mod->num_ctors; i++)
		mod->ctors[i]();
#endif
}

3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217
/* For freeing module_init on success, in case kallsyms traversing */
struct mod_initfree {
	struct rcu_head rcu;
	void *module_init;
};

static void do_free_init(struct rcu_head *head)
{
	struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
	module_memfree(m->module_init);
	kfree(m);
}

3218 3219 3220 3221 3222 3223 3224
/*
 * This is where the real work happens.
 *
 * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
 * helper command 'lx-symbols'.
 */
static noinline int do_init_module(struct module *mod)
3225 3226
{
	int ret = 0;
3227 3228 3229 3230 3231 3232 3233
	struct mod_initfree *freeinit;

	freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
	if (!freeinit) {
		ret = -ENOMEM;
		goto fail;
	}
3234
	freeinit->module_init = mod->init_layout.base;
3235

3236 3237 3238 3239 3240 3241
	/*
	 * We want to find out whether @mod uses async during init.  Clear
	 * PF_USED_ASYNC.  async_schedule*() will set it.
	 */
	current->flags &= ~PF_USED_ASYNC;

3242 3243 3244 3245 3246
	do_mod_ctors(mod);
	/* Start the module */
	if (mod->init != NULL)
		ret = do_one_initcall(mod->init);
	if (ret < 0) {
3247
		goto fail_free_freeinit;
3248 3249
	}
	if (ret > 0) {
A
Andrew Morton 已提交
3250 3251 3252 3253
		pr_warn("%s: '%s'->init suspiciously returned %d, it should "
			"follow 0/-E convention\n"
			"%s: loading module anyway...\n",
			__func__, mod->name, ret, __func__);
3254 3255 3256 3257 3258 3259 3260 3261
		dump_stack();
	}

	/* Now it's a first class citizen! */
	mod->state = MODULE_STATE_LIVE;
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_LIVE, mod);

3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278
	/*
	 * We need to finish all async code before the module init sequence
	 * is done.  This has potential to deadlock.  For example, a newly
	 * detected block device can trigger request_module() of the
	 * default iosched from async probing task.  Once userland helper
	 * reaches here, async_synchronize_full() will wait on the async
	 * task waiting on request_module() and deadlock.
	 *
	 * This deadlock is avoided by perfomring async_synchronize_full()
	 * iff module init queued any async jobs.  This isn't a full
	 * solution as it will deadlock the same if module loading from
	 * async jobs nests more than once; however, due to the various
	 * constraints, this hack seems to be the best option for now.
	 * Please refer to the following thread for details.
	 *
	 * http://thread.gmane.org/gmane.linux.kernel/1420814
	 */
3279
	if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC))
3280
		async_synchronize_full();
3281 3282 3283 3284 3285 3286

	mutex_lock(&module_mutex);
	/* Drop initial reference. */
	module_put(mod);
	trim_init_extable(mod);
#ifdef CONFIG_KALLSYMS
3287 3288
	/* Switch to core kallsyms now init is done: kallsyms may be walking! */
	rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
3289
#endif
3290
	mod_tree_remove_init(mod);
R
Rusty Russell 已提交
3291
	disable_ro_nx(&mod->init_layout);
3292
	module_arch_freeing_init(mod);
3293 3294 3295 3296
	mod->init_layout.base = NULL;
	mod->init_layout.size = 0;
	mod->init_layout.ro_size = 0;
	mod->init_layout.text_size = 0;
3297 3298
	/*
	 * We want to free module_init, but be aware that kallsyms may be
3299 3300 3301
	 * walking this with preempt disabled.  In all the failure paths, we
	 * call synchronize_sched(), but we don't want to slow down the success
	 * path, so use actual RCU here.
3302
	 */
3303
	call_rcu_sched(&freeinit->rcu, do_free_init);
3304 3305 3306 3307
	mutex_unlock(&module_mutex);
	wake_up_all(&module_wq);

	return 0;
3308 3309 3310 3311 3312 3313 3314 3315 3316 3317

fail_free_freeinit:
	kfree(freeinit);
fail:
	/* Try to protect us from buggy refcounters. */
	mod->state = MODULE_STATE_GOING;
	synchronize_sched();
	module_put(mod);
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);
3318
	ftrace_release_mod(mod);
3319 3320 3321
	free_module(mod);
	wake_up_all(&module_wq);
	return ret;
3322 3323 3324 3325 3326 3327 3328 3329 3330 3331
}

static int may_init_module(void)
{
	if (!capable(CAP_SYS_MODULE) || modules_disabled)
		return -EPERM;

	return 0;
}

3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
/*
 * We try to place it in the list now to make sure it's unique before
 * we dedicate too many resources.  In particular, temporary percpu
 * memory exhaustion.
 */
static int add_unformed_module(struct module *mod)
{
	int err;
	struct module *old;

	mod->state = MODULE_STATE_UNFORMED;

again:
	mutex_lock(&module_mutex);
3346 3347
	old = find_module_all(mod->name, strlen(mod->name), true);
	if (old != NULL) {
3348 3349 3350 3351
		if (old->state == MODULE_STATE_COMING
		    || old->state == MODULE_STATE_UNFORMED) {
			/* Wait in case it fails to load. */
			mutex_unlock(&module_mutex);
3352 3353
			err = wait_event_interruptible(module_wq,
					       finished_loading(mod->name));
3354 3355 3356 3357 3358 3359 3360
			if (err)
				goto out_unlocked;
			goto again;
		}
		err = -EEXIST;
		goto out;
	}
3361
	mod_update_bounds(mod);
3362
	list_add_rcu(&mod->list, &modules);
3363
	mod_tree_insert(mod);
3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385
	err = 0;

out:
	mutex_unlock(&module_mutex);
out_unlocked:
	return err;
}

static int complete_formation(struct module *mod, struct load_info *info)
{
	int err;

	mutex_lock(&module_mutex);

	/* Find duplicate symbols (must be called under lock). */
	err = verify_export_symbols(mod);
	if (err < 0)
		goto out;

	/* This relies on module_mutex for list integrity. */
	module_bug_finalize(info->hdr, info->sechdrs, mod);

3386
	/* Set RO and NX regions */
R
Rusty Russell 已提交
3387 3388
	module_enable_ro(mod);
	module_enable_nx(mod);
3389

3390 3391 3392
	/* Mark state as coming so strong_try_module_get() ignores us,
	 * but kallsyms etc. can see us. */
	mod->state = MODULE_STATE_COMING;
3393 3394 3395
	mutex_unlock(&module_mutex);

	return 0;
3396 3397 3398 3399 3400 3401

out:
	mutex_unlock(&module_mutex);
	return err;
}

3402 3403 3404 3405 3406 3407 3408 3409
static int prepare_coming_module(struct module *mod)
{
	ftrace_module_enable(mod);
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_COMING, mod);
	return 0;
}

3410 3411
static int unknown_module_param_cb(char *param, char *val, const char *modname,
				   void *arg)
3412
{
3413 3414 3415 3416 3417 3418 3419 3420
	struct module *mod = arg;
	int ret;

	if (strcmp(param, "async_probe") == 0) {
		mod->async_probe_requested = true;
		return 0;
	}

3421
	/* Check for magic 'dyndbg' arg */
3422
	ret = ddebug_dyndbg_module_param_cb(param, val, modname);
A
Andrew Morton 已提交
3423 3424
	if (ret != 0)
		pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
3425 3426 3427
	return 0;
}

R
Rusty Russell 已提交
3428 3429
/* Allocate and load the module: note that size of section 0 is always
   zero, and we rely on this for optional sections. */
3430 3431
static int load_module(struct load_info *info, const char __user *uargs,
		       int flags)
R
Rusty Russell 已提交
3432
{
3433
	struct module *mod;
R
Rusty Russell 已提交
3434
	long err;
3435
	char *after_dashes;
R
Rusty Russell 已提交
3436

3437 3438 3439
	err = module_sig_check(info);
	if (err)
		goto free_copy;
R
Rusty Russell 已提交
3440

3441
	err = elf_header_check(info);
R
Rusty Russell 已提交
3442
	if (err)
3443
		goto free_copy;
R
Rusty Russell 已提交
3444 3445

	/* Figure out module layout, and allocate all the memory. */
3446
	mod = layout_and_allocate(info, flags);
3447 3448
	if (IS_ERR(mod)) {
		err = PTR_ERR(mod);
R
Rusty Russell 已提交
3449
		goto free_copy;
L
Linus Torvalds 已提交
3450 3451
	}

3452 3453 3454
	/* Reserve our place in the list. */
	err = add_unformed_module(mod);
	if (err)
3455 3456
		goto free_module;

R
Rusty Russell 已提交
3457
#ifdef CONFIG_MODULE_SIG
3458
	mod->sig_ok = info->sig_ok;
3459
	if (!mod->sig_ok) {
A
Andrew Morton 已提交
3460
		pr_notice_once("%s: module verification failed: signature "
3461
			       "and/or required key missing - tainting "
A
Andrew Morton 已提交
3462
			       "kernel\n", mod->name);
3463
		add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
3464
	}
R
Rusty Russell 已提交
3465 3466
#endif

3467
	/* To avoid stressing percpu allocator, do this once we're unique. */
R
Rusty Russell 已提交
3468
	err = percpu_modalloc(mod, info);
3469 3470 3471
	if (err)
		goto unlink_mod;

3472
	/* Now module is in final location, initialize linked lists, etc. */
3473 3474
	err = module_unload_init(mod);
	if (err)
3475
		goto unlink_mod;
L
Linus Torvalds 已提交
3476

3477
	init_param_lock(mod);
3478

3479 3480
	/* Now we've got everything in the final locations, we can
	 * find optional sections. */
3481 3482 3483
	err = find_module_sections(mod, info);
	if (err)
		goto free_unload;
3484

3485
	err = check_module_license_and_versions(mod);
3486 3487
	if (err)
		goto free_unload;
3488

3489
	/* Set up MODINFO_ATTR fields */
3490
	setup_modinfo(mod, info);
3491

L
Linus Torvalds 已提交
3492
	/* Fix up syms, so that st_value is a pointer to location. */
3493
	err = simplify_symbols(mod, info);
L
Linus Torvalds 已提交
3494
	if (err < 0)
R
Rusty Russell 已提交
3495
		goto free_modinfo;
L
Linus Torvalds 已提交
3496

3497
	err = apply_relocations(mod, info);
3498
	if (err < 0)
R
Rusty Russell 已提交
3499
		goto free_modinfo;
L
Linus Torvalds 已提交
3500

3501
	err = post_relocation(mod, info);
L
Linus Torvalds 已提交
3502
	if (err < 0)
R
Rusty Russell 已提交
3503
		goto free_modinfo;
L
Linus Torvalds 已提交
3504

3505
	flush_module_icache(mod);
3506

3507 3508 3509 3510 3511 3512
	/* Now copy in args */
	mod->args = strndup_user(uargs, ~0UL >> 1);
	if (IS_ERR(mod->args)) {
		err = PTR_ERR(mod->args);
		goto free_arch_cleanup;
	}
R
Rusty Russell 已提交
3513

3514
	dynamic_debug_setup(info->debug, info->num_debug);
3515

3516 3517 3518
	/* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
	ftrace_module_init(mod);

3519 3520 3521
	/* Finally it's fully formed, ready to start executing. */
	err = complete_formation(mod, info);
	if (err)
3522
		goto ddebug_cleanup;
3523

3524 3525 3526 3527
	err = prepare_coming_module(mod);
	if (err)
		goto bug_cleanup;

3528
	/* Module is ready to execute: parsing args may do that. */
3529
	after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
3530
				  -32768, 32767, mod,
3531
				  unknown_module_param_cb);
3532 3533
	if (IS_ERR(after_dashes)) {
		err = PTR_ERR(after_dashes);
3534
		goto coming_cleanup;
3535 3536 3537 3538
	} else if (after_dashes) {
		pr_warn("%s: parameters '%s' after `--' ignored\n",
		       mod->name, after_dashes);
	}
L
Linus Torvalds 已提交
3539

3540
	/* Link in to syfs. */
3541
	err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
L
Linus Torvalds 已提交
3542
	if (err < 0)
3543
		goto coming_cleanup;
3544

3545
	/* Get rid of temporary copy. */
3546
	free_copy(info);
L
Linus Torvalds 已提交
3547 3548

	/* Done! */
3549
	trace_module_load(mod);
3550 3551

	return do_init_module(mod);
L
Linus Torvalds 已提交
3552

3553 3554 3555 3556
 coming_cleanup:
	blocking_notifier_call_chain(&module_notify_list,
				     MODULE_STATE_GOING, mod);

3557 3558
 bug_cleanup:
	/* module_bug_cleanup needs module_mutex protection */
3559
	mutex_lock(&module_mutex);
3560
	module_bug_cleanup(mod);
3561
	mutex_unlock(&module_mutex);
3562 3563

	/* we can't deallocate the module until we clear memory protection */
R
Rusty Russell 已提交
3564 3565
	module_disable_ro(mod);
	module_disable_nx(mod);
3566

3567
 ddebug_cleanup:
3568
	dynamic_debug_remove(info->debug);
3569
	synchronize_sched();
3570 3571
	kfree(mod->args);
 free_arch_cleanup:
L
Linus Torvalds 已提交
3572
	module_arch_cleanup(mod);
R
Rusty Russell 已提交
3573
 free_modinfo:
3574
	free_modinfo(mod);
3575
 free_unload:
L
Linus Torvalds 已提交
3576
	module_unload_free(mod);
3577 3578 3579 3580
 unlink_mod:
	mutex_lock(&module_mutex);
	/* Unlink carefully: kallsyms could be walking list. */
	list_del_rcu(&mod->list);
3581
	mod_tree_remove(mod);
3582
	wake_up_all(&module_wq);
3583 3584
	/* Wait for RCU-sched synchronizing before releasing mod->list. */
	synchronize_sched();
3585
	mutex_unlock(&module_mutex);
R
Rusty Russell 已提交
3586
 free_module:
3587 3588 3589 3590 3591 3592
	/*
	 * Ftrace needs to clean up what it initialized.
	 * This does nothing if ftrace_module_init() wasn't called,
	 * but it must be called outside of module_mutex.
	 */
	ftrace_release_mod(mod);
3593
	/* Free lock-classes; relies on the preceding sync_rcu() */
3594
	lockdep_free_key_range(mod->core_layout.base, mod->core_layout.size);
3595

3596
	module_deallocate(mod, info);
R
Rusty Russell 已提交
3597
 free_copy:
3598 3599
	free_copy(info);
	return err;
3600 3601
}

3602 3603
SYSCALL_DEFINE3(init_module, void __user *, umod,
		unsigned long, len, const char __user *, uargs)
L
Linus Torvalds 已提交
3604
{
3605 3606
	int err;
	struct load_info info = { };
L
Linus Torvalds 已提交
3607

3608 3609 3610
	err = may_init_module();
	if (err)
		return err;
L
Linus Torvalds 已提交
3611

3612 3613
	pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
	       umod, len, uargs);
L
Linus Torvalds 已提交
3614

3615 3616 3617
	err = copy_module_from_user(umod, len, &info);
	if (err)
		return err;
L
Linus Torvalds 已提交
3618

3619
	return load_module(&info, uargs, 0);
3620
}
3621

3622
SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
3623 3624 3625
{
	int err;
	struct load_info info = { };
3626

3627 3628 3629
	err = may_init_module();
	if (err)
		return err;
L
Linus Torvalds 已提交
3630

3631
	pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3632

3633 3634 3635
	if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
		      |MODULE_INIT_IGNORE_VERMAGIC))
		return -EINVAL;
3636

3637 3638 3639
	err = copy_module_from_fd(fd, &info);
	if (err)
		return err;
L
Linus Torvalds 已提交
3640

3641
	return load_module(&info, uargs, flags);
L
Linus Torvalds 已提交
3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655
}

static inline int within(unsigned long addr, void *start, unsigned long size)
{
	return ((void *)addr >= start && (void *)addr < start + size);
}

#ifdef CONFIG_KALLSYMS
/*
 * This ignores the intensely annoying "mapping symbols" found
 * in ARM ELF files: $a, $t and $d.
 */
static inline int is_arm_mapping_symbol(const char *str)
{
3656 3657
	if (str[0] == '.' && str[1] == 'L')
		return true;
K
Kyle McMartin 已提交
3658
	return str[0] == '$' && strchr("axtd", str[1])
L
Linus Torvalds 已提交
3659 3660 3661
	       && (str[2] == '\0' || str[2] == '.');
}

3662
static const char *symname(struct mod_kallsyms *kallsyms, unsigned int symnum)
R
Rusty Russell 已提交
3663
{
3664
	return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
R
Rusty Russell 已提交
3665 3666
}

L
Linus Torvalds 已提交
3667 3668 3669 3670 3671 3672 3673
static const char *get_ksymbol(struct module *mod,
			       unsigned long addr,
			       unsigned long *size,
			       unsigned long *offset)
{
	unsigned int i, best = 0;
	unsigned long nextval;
3674
	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
L
Linus Torvalds 已提交
3675 3676

	/* At worse, next value is at end of module */
3677
	if (within_module_init(addr, mod))
3678
		nextval = (unsigned long)mod->init_layout.base+mod->init_layout.text_size;
D
Daniel Walker 已提交
3679
	else
3680
		nextval = (unsigned long)mod->core_layout.base+mod->core_layout.text_size;
L
Linus Torvalds 已提交
3681

L
Lucas De Marchi 已提交
3682
	/* Scan for closest preceding symbol, and next symbol. (ELF
D
Daniel Walker 已提交
3683
	   starts real symbols at 1). */
3684 3685
	for (i = 1; i < kallsyms->num_symtab; i++) {
		if (kallsyms->symtab[i].st_shndx == SHN_UNDEF)
L
Linus Torvalds 已提交
3686 3687 3688 3689
			continue;

		/* We ignore unnamed symbols: they're uninformative
		 * and inserted at a whim. */
3690 3691
		if (*symname(kallsyms, i) == '\0'
		    || is_arm_mapping_symbol(symname(kallsyms, i)))
R
Rusty Russell 已提交
3692 3693
			continue;

3694 3695
		if (kallsyms->symtab[i].st_value <= addr
		    && kallsyms->symtab[i].st_value > kallsyms->symtab[best].st_value)
L
Linus Torvalds 已提交
3696
			best = i;
3697 3698 3699
		if (kallsyms->symtab[i].st_value > addr
		    && kallsyms->symtab[i].st_value < nextval)
			nextval = kallsyms->symtab[i].st_value;
L
Linus Torvalds 已提交
3700 3701 3702 3703 3704
	}

	if (!best)
		return NULL;

A
Alexey Dobriyan 已提交
3705
	if (size)
3706
		*size = nextval - kallsyms->symtab[best].st_value;
A
Alexey Dobriyan 已提交
3707
	if (offset)
3708 3709
		*offset = addr - kallsyms->symtab[best].st_value;
	return symname(kallsyms, best);
L
Linus Torvalds 已提交
3710 3711
}

3712 3713
/* For kallsyms to ask for address resolution.  NULL means not found.  Careful
 * not to lock to avoid deadlock on oopses, simply disable preemption. */
3714
const char *module_address_lookup(unsigned long addr,
3715 3716 3717 3718
			    unsigned long *size,
			    unsigned long *offset,
			    char **modname,
			    char *namebuf)
L
Linus Torvalds 已提交
3719
{
3720
	const char *ret = NULL;
3721
	struct module *mod;
L
Linus Torvalds 已提交
3722

3723
	preempt_disable();
3724 3725 3726 3727 3728
	mod = __module_address(addr);
	if (mod) {
		if (modname)
			*modname = mod->name;
		ret = get_ksymbol(mod, addr, size, offset);
L
Linus Torvalds 已提交
3729
	}
3730 3731 3732 3733 3734
	/* Make a copy in here where it's safe */
	if (ret) {
		strncpy(namebuf, ret, KSYM_NAME_LEN - 1);
		ret = namebuf;
	}
3735
	preempt_enable();
3736

3737
	return ret;
L
Linus Torvalds 已提交
3738 3739
}

3740 3741 3742 3743
int lookup_module_symbol_name(unsigned long addr, char *symname)
{
	struct module *mod;

3744
	preempt_disable();
3745
	list_for_each_entry_rcu(mod, &modules, list) {
3746 3747
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
3748
		if (within_module(addr, mod)) {
3749 3750 3751 3752 3753
			const char *sym;

			sym = get_ksymbol(mod, addr, NULL, NULL);
			if (!sym)
				goto out;
3754
			strlcpy(symname, sym, KSYM_NAME_LEN);
3755
			preempt_enable();
3756 3757 3758 3759
			return 0;
		}
	}
out:
3760
	preempt_enable();
3761 3762 3763
	return -ERANGE;
}

3764 3765 3766 3767 3768
int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
			unsigned long *offset, char *modname, char *name)
{
	struct module *mod;

3769
	preempt_disable();
3770
	list_for_each_entry_rcu(mod, &modules, list) {
3771 3772
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
3773
		if (within_module(addr, mod)) {
3774 3775 3776 3777 3778 3779
			const char *sym;

			sym = get_ksymbol(mod, addr, size, offset);
			if (!sym)
				goto out;
			if (modname)
3780
				strlcpy(modname, mod->name, MODULE_NAME_LEN);
3781
			if (name)
3782
				strlcpy(name, sym, KSYM_NAME_LEN);
3783
			preempt_enable();
3784 3785 3786 3787
			return 0;
		}
	}
out:
3788
	preempt_enable();
3789 3790 3791
	return -ERANGE;
}

3792 3793
int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
			char *name, char *module_name, int *exported)
L
Linus Torvalds 已提交
3794 3795 3796
{
	struct module *mod;

3797
	preempt_disable();
3798
	list_for_each_entry_rcu(mod, &modules, list) {
3799 3800
		struct mod_kallsyms *kallsyms;

3801 3802
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
3803 3804 3805 3806 3807
		kallsyms = rcu_dereference_sched(mod->kallsyms);
		if (symnum < kallsyms->num_symtab) {
			*value = kallsyms->symtab[symnum].st_value;
			*type = kallsyms->symtab[symnum].st_info;
			strlcpy(name, symname(kallsyms, symnum), KSYM_NAME_LEN);
3808
			strlcpy(module_name, mod->name, MODULE_NAME_LEN);
3809
			*exported = is_exported(name, *value, mod);
3810
			preempt_enable();
3811
			return 0;
L
Linus Torvalds 已提交
3812
		}
3813
		symnum -= kallsyms->num_symtab;
L
Linus Torvalds 已提交
3814
	}
3815
	preempt_enable();
3816
	return -ERANGE;
L
Linus Torvalds 已提交
3817 3818 3819 3820 3821
}

static unsigned long mod_find_symname(struct module *mod, const char *name)
{
	unsigned int i;
3822
	struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
L
Linus Torvalds 已提交
3823

3824 3825 3826 3827
	for (i = 0; i < kallsyms->num_symtab; i++)
		if (strcmp(name, symname(kallsyms, i)) == 0 &&
		    kallsyms->symtab[i].st_info != 'U')
			return kallsyms->symtab[i].st_value;
L
Linus Torvalds 已提交
3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838
	return 0;
}

/* Look for this name: can be of form module:name. */
unsigned long module_kallsyms_lookup_name(const char *name)
{
	struct module *mod;
	char *colon;
	unsigned long ret = 0;

	/* Don't lock: we're in enough trouble already. */
3839
	preempt_disable();
L
Linus Torvalds 已提交
3840
	if ((colon = strchr(name, ':')) != NULL) {
3841
		if ((mod = find_module_all(name, colon - name, false)) != NULL)
L
Linus Torvalds 已提交
3842 3843
			ret = mod_find_symname(mod, colon+1);
	} else {
3844 3845 3846
		list_for_each_entry_rcu(mod, &modules, list) {
			if (mod->state == MODULE_STATE_UNFORMED)
				continue;
L
Linus Torvalds 已提交
3847 3848
			if ((ret = mod_find_symname(mod, name)) != 0)
				break;
3849
		}
L
Linus Torvalds 已提交
3850
	}
3851
	preempt_enable();
L
Linus Torvalds 已提交
3852 3853
	return ret;
}
3854 3855 3856 3857 3858 3859 3860 3861 3862

int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
					     struct module *, unsigned long),
				   void *data)
{
	struct module *mod;
	unsigned int i;
	int ret;

3863 3864
	module_assert_mutex();

3865
	list_for_each_entry(mod, &modules, list) {
3866 3867 3868
		/* We hold module_mutex: no need for rcu_dereference_sched */
		struct mod_kallsyms *kallsyms = mod->kallsyms;

3869 3870
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
3871 3872 3873
		for (i = 0; i < kallsyms->num_symtab; i++) {
			ret = fn(data, symname(kallsyms, i),
				 mod, kallsyms->symtab[i].st_value);
3874 3875 3876 3877 3878 3879
			if (ret != 0)
				return ret;
		}
	}
	return 0;
}
L
Linus Torvalds 已提交
3880 3881
#endif /* CONFIG_KALLSYMS */

3882
static char *module_flags(struct module *mod, char *buf)
3883 3884 3885
{
	int bx = 0;

3886
	BUG_ON(mod->state == MODULE_STATE_UNFORMED);
3887 3888 3889
	if (mod->taints ||
	    mod->state == MODULE_STATE_GOING ||
	    mod->state == MODULE_STATE_COMING) {
3890
		buf[bx++] = '(';
3891
		bx += module_flags_taint(mod, buf + bx);
3892 3893 3894 3895 3896 3897
		/* Show a - for module-is-being-unloaded */
		if (mod->state == MODULE_STATE_GOING)
			buf[bx++] = '-';
		/* Show a + for module-is-being-loaded */
		if (mod->state == MODULE_STATE_COMING)
			buf[bx++] = '+';
3898 3899 3900 3901 3902 3903 3904
		buf[bx++] = ')';
	}
	buf[bx] = '\0';

	return buf;
}

3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922
#ifdef CONFIG_PROC_FS
/* Called by the /proc file system to return a list of modules. */
static void *m_start(struct seq_file *m, loff_t *pos)
{
	mutex_lock(&module_mutex);
	return seq_list_start(&modules, *pos);
}

static void *m_next(struct seq_file *m, void *p, loff_t *pos)
{
	return seq_list_next(p, &modules, pos);
}

static void m_stop(struct seq_file *m, void *p)
{
	mutex_unlock(&module_mutex);
}

L
Linus Torvalds 已提交
3923 3924 3925
static int m_show(struct seq_file *m, void *p)
{
	struct module *mod = list_entry(p, struct module, list);
3926 3927
	char buf[8];

3928 3929 3930 3931
	/* We always ignore unformed modules. */
	if (mod->state == MODULE_STATE_UNFORMED)
		return 0;

3932
	seq_printf(m, "%s %u",
3933
		   mod->name, mod->init_layout.size + mod->core_layout.size);
L
Linus Torvalds 已提交
3934 3935 3936 3937
	print_unload_info(m, mod);

	/* Informative for users. */
	seq_printf(m, " %s",
3938 3939
		   mod->state == MODULE_STATE_GOING ? "Unloading" :
		   mod->state == MODULE_STATE_COMING ? "Loading" :
L
Linus Torvalds 已提交
3940 3941
		   "Live");
	/* Used by oprofile and other similar tools. */
3942
	seq_printf(m, " 0x%pK", mod->core_layout.base);
L
Linus Torvalds 已提交
3943

3944 3945
	/* Taints info */
	if (mod->taints)
3946
		seq_printf(m, " %s", module_flags(mod, buf));
3947

3948
	seq_puts(m, "\n");
L
Linus Torvalds 已提交
3949 3950 3951 3952 3953 3954 3955 3956
	return 0;
}

/* Format: modulename size refcount deps address

   Where refcount is a number or -, and deps is a comma-separated list
   of depends or -.
*/
3957
static const struct seq_operations modules_op = {
L
Linus Torvalds 已提交
3958 3959 3960 3961 3962 3963
	.start	= m_start,
	.next	= m_next,
	.stop	= m_stop,
	.show	= m_show
};

3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
static int modules_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &modules_op);
}

static const struct file_operations proc_modules_operations = {
	.open		= modules_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init proc_modules_init(void)
{
	proc_create("modules", 0, NULL, &proc_modules_operations);
	return 0;
}
module_init(proc_modules_init);
#endif

L
Linus Torvalds 已提交
3984 3985 3986 3987 3988 3989
/* Given an address, look for it in the module exception tables. */
const struct exception_table_entry *search_module_extables(unsigned long addr)
{
	const struct exception_table_entry *e = NULL;
	struct module *mod;

R
Rusty Russell 已提交
3990
	preempt_disable();
3991
	list_for_each_entry_rcu(mod, &modules, list) {
3992 3993
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
L
Linus Torvalds 已提交
3994 3995
		if (mod->num_exentries == 0)
			continue;
D
Daniel Walker 已提交
3996

L
Linus Torvalds 已提交
3997 3998 3999 4000 4001 4002
		e = search_extable(mod->extable,
				   mod->extable + mod->num_exentries - 1,
				   addr);
		if (e)
			break;
	}
R
Rusty Russell 已提交
4003
	preempt_enable();
L
Linus Torvalds 已提交
4004 4005

	/* Now, if we found one, we are running inside it now, hence
D
Daniel Walker 已提交
4006
	   we cannot unload the module, hence no refcnt needed. */
L
Linus Torvalds 已提交
4007 4008 4009
	return e;
}

4010
/*
R
Rusty Russell 已提交
4011 4012 4013 4014 4015
 * is_module_address - is this address inside a module?
 * @addr: the address to check.
 *
 * See is_module_text_address() if you simply want to see if the address
 * is code (not data).
4016
 */
R
Rusty Russell 已提交
4017
bool is_module_address(unsigned long addr)
4018
{
R
Rusty Russell 已提交
4019
	bool ret;
4020

R
Rusty Russell 已提交
4021
	preempt_disable();
R
Rusty Russell 已提交
4022
	ret = __module_address(addr) != NULL;
R
Rusty Russell 已提交
4023
	preempt_enable();
4024

R
Rusty Russell 已提交
4025
	return ret;
4026 4027
}

R
Rusty Russell 已提交
4028 4029 4030 4031 4032 4033 4034
/*
 * __module_address - get the module which contains an address.
 * @addr: the address.
 *
 * Must be called with preempt disabled or module mutex held so that
 * module doesn't get freed during this.
 */
4035
struct module *__module_address(unsigned long addr)
L
Linus Torvalds 已提交
4036 4037 4038
{
	struct module *mod;

4039 4040 4041
	if (addr < module_addr_min || addr > module_addr_max)
		return NULL;

4042 4043
	module_assert_mutex_or_preempt();

4044
	mod = mod_find(addr);
4045 4046
	if (mod) {
		BUG_ON(!within_module(addr, mod));
4047
		if (mod->state == MODULE_STATE_UNFORMED)
4048
			mod = NULL;
4049
	}
4050
	return mod;
L
Linus Torvalds 已提交
4051
}
4052
EXPORT_SYMBOL_GPL(__module_address);
L
Linus Torvalds 已提交
4053

R
Rusty Russell 已提交
4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084
/*
 * is_module_text_address - is this address inside module code?
 * @addr: the address to check.
 *
 * See is_module_address() if you simply want to see if the address is
 * anywhere in a module.  See kernel_text_address() for testing if an
 * address corresponds to kernel or module code.
 */
bool is_module_text_address(unsigned long addr)
{
	bool ret;

	preempt_disable();
	ret = __module_text_address(addr) != NULL;
	preempt_enable();

	return ret;
}

/*
 * __module_text_address - get the module whose code contains an address.
 * @addr: the address.
 *
 * Must be called with preempt disabled or module mutex held so that
 * module doesn't get freed during this.
 */
struct module *__module_text_address(unsigned long addr)
{
	struct module *mod = __module_address(addr);
	if (mod) {
		/* Make sure it's within the text section. */
4085 4086
		if (!within(addr, mod->init_layout.base, mod->init_layout.text_size)
		    && !within(addr, mod->core_layout.base, mod->core_layout.text_size))
R
Rusty Russell 已提交
4087 4088 4089 4090
			mod = NULL;
	}
	return mod;
}
4091
EXPORT_SYMBOL_GPL(__module_text_address);
R
Rusty Russell 已提交
4092

L
Linus Torvalds 已提交
4093 4094 4095 4096
/* Don't grab lock, we're oopsing. */
void print_modules(void)
{
	struct module *mod;
4097
	char buf[8];
L
Linus Torvalds 已提交
4098

4099
	printk(KERN_DEFAULT "Modules linked in:");
4100 4101
	/* Most callers should already have preempt disabled, but make sure */
	preempt_disable();
4102 4103 4104
	list_for_each_entry_rcu(mod, &modules, list) {
		if (mod->state == MODULE_STATE_UNFORMED)
			continue;
J
Jiri Slaby 已提交
4105
		pr_cont(" %s%s", mod->name, module_flags(mod, buf));
4106
	}
4107
	preempt_enable();
4108
	if (last_unloaded_module[0])
J
Jiri Slaby 已提交
4109 4110
		pr_cont(" [last unloaded: %s]", last_unloaded_module);
	pr_cont("\n");
L
Linus Torvalds 已提交
4111 4112 4113
}

#ifdef CONFIG_MODVERSIONS
4114 4115 4116 4117 4118 4119
/* Generate the signature for all relevant module structures here.
 * If these change, we don't want to try to parse the module. */
void module_layout(struct module *mod,
		   struct modversion_info *ver,
		   struct kernel_param *kp,
		   struct kernel_symbol *ks,
4120
		   struct tracepoint * const *tp)
4121 4122 4123
{
}
EXPORT_SYMBOL(module_layout);
L
Linus Torvalds 已提交
4124
#endif