main.c 22.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 *  linux/init/main.c
 *
 *  Copyright (C) 1991, 1992  Linus Torvalds
 *
 *  GK 2/5/95  -  Changed to support mounting root fs via NFS
 *  Added initrd & change_root: Werner Almesberger & Hans Lermen, Feb '96
 *  Moan early if gcc is old, avoiding bogus kernels - Paul Gortmaker, May '96
 *  Simplified starting of init:  Michael A. Griffith <grif@acm.org> 
 */

#include <linux/types.h>
#include <linux/module.h>
#include <linux/proc_fs.h>
#include <linux/kernel.h>
#include <linux/syscalls.h>
I
Ingo Molnar 已提交
17
#include <linux/stackprotector.h>
L
Linus Torvalds 已提交
18 19 20 21 22 23 24 25 26
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/delay.h>
#include <linux/utsname.h>
#include <linux/ioport.h>
#include <linux/init.h>
#include <linux/smp_lock.h>
#include <linux/initrd.h>
#include <linux/bootmem.h>
27
#include <linux/acpi.h>
L
Linus Torvalds 已提交
28 29 30 31
#include <linux/tty.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
#include <linux/kmod.h>
N
Nick Piggin 已提交
32
#include <linux/vmalloc.h>
L
Linus Torvalds 已提交
33
#include <linux/kernel_stat.h>
34
#include <linux/start_kernel.h>
L
Linus Torvalds 已提交
35
#include <linux/security.h>
36
#include <linux/smp.h>
L
Linus Torvalds 已提交
37 38 39 40 41 42 43 44
#include <linux/workqueue.h>
#include <linux/profile.h>
#include <linux/rcupdate.h>
#include <linux/moduleparam.h>
#include <linux/kallsyms.h>
#include <linux/writeback.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
45
#include <linux/cgroup.h>
L
Linus Torvalds 已提交
46
#include <linux/efi.h>
47
#include <linux/tick.h>
48
#include <linux/interrupt.h>
49
#include <linux/taskstats_kern.h>
50
#include <linux/delayacct.h>
L
Linus Torvalds 已提交
51 52 53 54
#include <linux/unistd.h>
#include <linux/rmap.h>
#include <linux/mempolicy.h>
#include <linux/key.h>
A
Adrian Bunk 已提交
55
#include <linux/buffer_head.h>
56
#include <linux/page_cgroup.h>
57
#include <linux/debug_locks.h>
58
#include <linux/debugobjects.h>
I
Ingo Molnar 已提交
59
#include <linux/lockdep.h>
C
Catalin Marinas 已提交
60
#include <linux/kmemleak.h>
61
#include <linux/pid_namespace.h>
62
#include <linux/device.h>
63
#include <linux/kthread.h>
64
#include <linux/sched.h>
65
#include <linux/signal.h>
66
#include <linux/idr.h>
67
#include <linux/ftrace.h>
68
#include <linux/async.h>
69
#include <linux/kmemcheck.h>
70
#include <linux/kmemtrace.h>
71
#include <trace/boot.h>
L
Linus Torvalds 已提交
72 73 74 75

#include <asm/io.h>
#include <asm/bugs.h>
#include <asm/setup.h>
76
#include <asm/sections.h>
77
#include <asm/cacheflush.h>
L
Linus Torvalds 已提交
78 79 80 81 82

#ifdef CONFIG_X86_LOCAL_APIC
#include <asm/smp.h>
#endif

83
static int kernel_init(void *);
L
Linus Torvalds 已提交
84 85 86 87 88 89 90 91

extern void init_IRQ(void);
extern void fork_init(unsigned long);
extern void mca_init(void);
extern void sbus_init(void);
extern void prio_tree_init(void);
extern void radix_tree_init(void);
extern void free_initmem(void);
92 93 94
#ifndef CONFIG_DEBUG_RODATA
static inline void mark_rodata_ro(void) { }
#endif
L
Linus Torvalds 已提交
95 96 97 98 99

#ifdef CONFIG_TC
extern void tc_init(void);
#endif

100
enum system_states system_state __read_mostly;
L
Linus Torvalds 已提交
101 102 103 104 105 106 107 108 109 110
EXPORT_SYMBOL(system_state);

/*
 * Boot command-line arguments
 */
#define MAX_INIT_ARGS CONFIG_INIT_ENV_ARG_LIMIT
#define MAX_INIT_ENVS CONFIG_INIT_ENV_ARG_LIMIT

extern void time_init(void);
/* Default late time init is NULL. archs can override this later. */
111
void (*__initdata late_time_init)(void);
L
Linus Torvalds 已提交
112 113
extern void softirq_init(void);

114 115 116 117 118 119
/* Untouched command line saved by arch-specific code. */
char __initdata boot_command_line[COMMAND_LINE_SIZE];
/* Untouched saved command line (eg. for /proc) */
char *saved_command_line;
/* Command line for parameter parsing */
static char *static_command_line;
L
Linus Torvalds 已提交
120 121

static char *execute_command;
122
static char *ramdisk_execute_command;
L
Linus Torvalds 已提交
123

J
Jan Beulich 已提交
124
#ifdef CONFIG_SMP
L
Linus Torvalds 已提交
125
/* Setup configured maximum number of CPUs to activate */
126
unsigned int __initdata setup_max_cpus = NR_CPUS;
127

L
Linus Torvalds 已提交
128 129 130 131 132 133 134 135 136 137
/*
 * Setup routine for controlling SMP activation
 *
 * Command-line option of "nosmp" or "maxcpus=0" will disable SMP
 * activation entirely (the MPS table probe still happens, though).
 *
 * Command-line option of "maxcpus=<NUM>", where <NUM> is an integer
 * greater than 0, limits the maximum number of CPUs activated in
 * SMP mode to <NUM>.
 */
138 139

void __weak arch_disable_smp_support(void) { }
140

L
Linus Torvalds 已提交
141 142
static int __init nosmp(char *str)
{
143
	setup_max_cpus = 0;
144 145
	arch_disable_smp_support();

J
Jan Beulich 已提交
146
	return 0;
L
Linus Torvalds 已提交
147 148
}

J
Jan Beulich 已提交
149
early_param("nosmp", nosmp);
L
Linus Torvalds 已提交
150 151 152

static int __init maxcpus(char *str)
{
153 154
	get_option(&str, &setup_max_cpus);
	if (setup_max_cpus == 0)
155
		arch_disable_smp_support();
156 157

	return 0;
L
Linus Torvalds 已提交
158 159
}

H
Hugh Dickins 已提交
160
early_param("maxcpus", maxcpus);
J
Jan Beulich 已提交
161
#else
162
const unsigned int setup_max_cpus = NR_CPUS;
J
Jan Beulich 已提交
163 164 165 166 167 168 169 170 171 172 173 174 175
#endif

/*
 * If set, this is an indication to the drivers that reset the underlying
 * device before going ahead with the initialization otherwise driver might
 * rely on the BIOS and skip the reset operation.
 *
 * This is useful if kernel is booting in an unreliable environment.
 * For ex. kdump situaiton where previous kernel has crashed, BIOS has been
 * skipped and devices will be in unknown state.
 */
unsigned int reset_devices;
EXPORT_SYMBOL(reset_devices);
L
Linus Torvalds 已提交
176

177 178 179 180 181 182 183 184
static int __init set_reset_devices(char *str)
{
	reset_devices = 1;
	return 1;
}

__setup("reset_devices", set_reset_devices);

L
Linus Torvalds 已提交
185 186 187 188 189 190 191 192 193
static char * argv_init[MAX_INIT_ARGS+2] = { "init", NULL, };
char * envp_init[MAX_INIT_ENVS+2] = { "HOME=/", "TERM=linux", NULL, };
static const char *panic_later, *panic_param;

extern struct obs_kernel_param __setup_start[], __setup_end[];

static int __init obsolete_checksetup(char *line)
{
	struct obs_kernel_param *p;
194
	int had_early_param = 0;
L
Linus Torvalds 已提交
195 196 197 198 199 200

	p = __setup_start;
	do {
		int n = strlen(p->str);
		if (!strncmp(line, p->str, n)) {
			if (p->early) {
201 202 203 204
				/* Already done in parse_early_param?
				 * (Needs exact match on param part).
				 * Keep iterating, as we can have early
				 * params and __setups of same names 8( */
L
Linus Torvalds 已提交
205
				if (line[n] == '\0' || line[n] == '=')
206
					had_early_param = 1;
L
Linus Torvalds 已提交
207 208 209 210 211 212 213 214 215
			} else if (!p->setup_func) {
				printk(KERN_WARNING "Parameter %s is obsolete,"
				       " ignored\n", p->str);
				return 1;
			} else if (p->setup_func(line + n))
				return 1;
		}
		p++;
	} while (p < __setup_end);
216 217

	return had_early_param;
L
Linus Torvalds 已提交
218 219 220 221 222 223 224 225 226 227 228 229 230
}

/*
 * This should be approx 2 Bo*oMips to start (note initial shift), and will
 * still work even if initially too large, it will just take slightly longer
 */
unsigned long loops_per_jiffy = (1<<12);

EXPORT_SYMBOL(loops_per_jiffy);

static int __init debug_kernel(char *str)
{
	console_loglevel = 10;
231
	return 0;
L
Linus Torvalds 已提交
232 233 234 235 236
}

static int __init quiet_kernel(char *str)
{
	console_loglevel = 4;
237
	return 0;
L
Linus Torvalds 已提交
238 239
}

240 241
early_param("debug", debug_kernel);
early_param("quiet", quiet_kernel);
L
Linus Torvalds 已提交
242 243 244 245

static int __init loglevel(char *str)
{
	get_option(&str, &console_loglevel);
246
	return 0;
L
Linus Torvalds 已提交
247 248
}

249
early_param("loglevel", loglevel);
L
Linus Torvalds 已提交
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274

/*
 * Unknown boot options get handed to init, unless they look like
 * failed parameters
 */
static int __init unknown_bootoption(char *param, char *val)
{
	/* Change NUL term back to "=", to make "param" the whole string. */
	if (val) {
		/* param=val or param="val"? */
		if (val == param+strlen(param)+1)
			val[-1] = '=';
		else if (val == param+strlen(param)+2) {
			val[-2] = '=';
			memmove(val-1, val, strlen(val)+1);
			val--;
		} else
			BUG();
	}

	/* Handle obsolete-style parameters */
	if (obsolete_checksetup(param))
		return 0;

	/*
S
Simon Arlott 已提交
275
	 * Preemptive maintenance for "why didn't my misspelled command
L
Linus Torvalds 已提交
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
	 * line work?"
	 */
	if (strchr(param, '.') && (!val || strchr(param, '.') < val)) {
		printk(KERN_ERR "Unknown boot option `%s': ignoring\n", param);
		return 0;
	}

	if (panic_later)
		return 0;

	if (val) {
		/* Environment option */
		unsigned int i;
		for (i = 0; envp_init[i]; i++) {
			if (i == MAX_INIT_ENVS) {
				panic_later = "Too many boot env vars at `%s'";
				panic_param = param;
			}
			if (!strncmp(param, envp_init[i], val - param))
				break;
		}
		envp_init[i] = param;
	} else {
		/* Command line option */
		unsigned int i;
		for (i = 0; argv_init[i]; i++) {
			if (i == MAX_INIT_ARGS) {
				panic_later = "Too many boot init vars at `%s'";
				panic_param = param;
			}
		}
		argv_init[i] = param;
	}
	return 0;
}

312 313 314 315
#ifdef CONFIG_DEBUG_PAGEALLOC
int __read_mostly debug_pagealloc_enabled = 0;
#endif

L
Linus Torvalds 已提交
316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
static int __init init_setup(char *str)
{
	unsigned int i;

	execute_command = str;
	/*
	 * In case LILO is going to boot us with default command line,
	 * it prepends "auto" before the whole cmdline which makes
	 * the shell think it should execute a script with such name.
	 * So we ignore all arguments entered _before_ init=... [MJ]
	 */
	for (i = 1; i < MAX_INIT_ARGS; i++)
		argv_init[i] = NULL;
	return 1;
}
__setup("init=", init_setup);

333 334 335 336 337 338 339 340 341 342 343 344
static int __init rdinit_setup(char *str)
{
	unsigned int i;

	ramdisk_execute_command = str;
	/* See "auto" comment in init_setup */
	for (i = 1; i < MAX_INIT_ARGS; i++)
		argv_init[i] = NULL;
	return 1;
}
__setup("rdinit=", rdinit_setup);

L
Linus Torvalds 已提交
345 346 347 348 349 350 351 352 353 354 355 356
#ifndef CONFIG_SMP

#ifdef CONFIG_X86_LOCAL_APIC
static void __init smp_init(void)
{
	APIC_init_uniprocessor();
}
#else
#define smp_init()	do { } while (0)
#endif

static inline void setup_per_cpu_areas(void) { }
357
static inline void setup_nr_cpu_ids(void) { }
L
Linus Torvalds 已提交
358 359 360 361
static inline void smp_prepare_cpus(unsigned int maxcpus) { }

#else

362 363 364 365 366
#if NR_CPUS > BITS_PER_LONG
cpumask_t cpu_mask_all __read_mostly = CPU_MASK_ALL;
EXPORT_SYMBOL(cpu_mask_all);
#endif

367 368 369 370 371 372 373
/* Setup number of possible processor ids */
int nr_cpu_ids __read_mostly = NR_CPUS;
EXPORT_SYMBOL(nr_cpu_ids);

/* An arch may set nr_cpu_ids earlier if needed, so this would be redundant */
static void __init setup_nr_cpu_ids(void)
{
R
Rusty Russell 已提交
374
	nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
375 376
}

377
#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
E
Eric Dumazet 已提交
378
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
L
Linus Torvalds 已提交
379 380 381 382 383 384 385

EXPORT_SYMBOL(__per_cpu_offset);

static void __init setup_per_cpu_areas(void)
{
	unsigned long size, i;
	char *ptr;
386
	unsigned long nr_possible_cpus = num_possible_cpus();
L
Linus Torvalds 已提交
387 388

	/* Copy section for each CPU (we discard the original) */
389 390
	size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
	ptr = alloc_bootmem_pages(size * nr_possible_cpus);
L
Linus Torvalds 已提交
391

392
	for_each_possible_cpu(i) {
L
Linus Torvalds 已提交
393 394
		__per_cpu_offset[i] = ptr - __per_cpu_start;
		memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
395
		ptr += size;
L
Linus Torvalds 已提交
396 397
	}
}
398
#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
L
Linus Torvalds 已提交
399 400 401 402

/* Called by boot processor to activate the rest. */
static void __init smp_init(void)
{
403
	unsigned int cpu;
L
Linus Torvalds 已提交
404

405 406 407 408
	/*
	 * Set up the current CPU as possible to migrate to.
	 * The other ones will be done by cpu_up/cpu_down()
	 */
409
	set_cpu_active(smp_processor_id(), true);
410

L
Linus Torvalds 已提交
411
	/* FIXME: This should be done in userspace --RR */
412
	for_each_present_cpu(cpu) {
413
		if (num_online_cpus() >= setup_max_cpus)
L
Linus Torvalds 已提交
414
			break;
415 416
		if (!cpu_online(cpu))
			cpu_up(cpu);
L
Linus Torvalds 已提交
417 418 419 420
	}

	/* Any cleanup work */
	printk(KERN_INFO "Brought up %ld CPUs\n", (long)num_online_cpus());
421
	smp_cpus_done(setup_max_cpus);
L
Linus Torvalds 已提交
422 423 424 425
}

#endif

426 427 428 429 430 431 432 433 434 435 436 437 438 439
/*
 * We need to store the untouched command line for future reference.
 * We also need to store the touched command line since the parameter
 * parsing is performed in place, and we should allow a component to
 * store reference of name/value for future reference.
 */
static void __init setup_command_line(char *command_line)
{
	saved_command_line = alloc_bootmem(strlen (boot_command_line)+1);
	static_command_line = alloc_bootmem(strlen (command_line)+1);
	strcpy (saved_command_line, boot_command_line);
	strcpy (static_command_line, command_line);
}

L
Linus Torvalds 已提交
440 441 442 443 444 445 446 447 448
/*
 * We need to finalize in a non-__init function or else race conditions
 * between the root thread and the init thread may cause start_kernel to
 * be reaped by free_initmem before the root thread has proceeded to
 * cpu_idle.
 *
 * gcc-3.4 accidentally inlines this function, so use noinline.
 */

449
static noinline void __init_refok rest_init(void)
L
Linus Torvalds 已提交
450 451
	__releases(kernel_lock)
{
452 453
	int pid;

454
	kernel_thread(kernel_init, NULL, CLONE_FS | CLONE_SIGHAND);
L
Linus Torvalds 已提交
455
	numa_default_policy();
456
	pid = kernel_thread(kthreadd, NULL, CLONE_FS | CLONE_FILES);
P
Pavel Emelyanov 已提交
457
	kthreadd_task = find_task_by_pid_ns(pid, &init_pid_ns);
L
Linus Torvalds 已提交
458
	unlock_kernel();
459 460 461

	/*
	 * The boot idle thread must execute schedule()
I
Ingo Molnar 已提交
462
	 * at least once to get things moving:
463
	 */
I
Ingo Molnar 已提交
464
	init_idle_bootup_task(current);
465
	rcu_scheduler_starting();
466
	preempt_enable_no_resched();
467
	schedule();
468
	preempt_disable();
469

470
	/* Call into cpu_idle with preempt disabled */
L
Linus Torvalds 已提交
471
	cpu_idle();
I
Ingo Molnar 已提交
472
}
L
Linus Torvalds 已提交
473 474 475 476 477 478 479

/* Check for early params. */
static int __init do_early_param(char *param, char *val)
{
	struct obs_kernel_param *p;

	for (p = __setup_start; p < __setup_end; p++) {
480 481 482 483
		if ((p->early && strcmp(param, p->str) == 0) ||
		    (strcmp(param, "console") == 0 &&
		     strcmp(p->str, "earlycon") == 0)
		) {
L
Linus Torvalds 已提交
484 485 486 487 488 489 490 491 492
			if (p->setup_func(val) != 0)
				printk(KERN_WARNING
				       "Malformed early option '%s'\n", param);
		}
	}
	/* We accept everything at this stage. */
	return 0;
}

M
Magnus Damm 已提交
493 494 495 496 497
void __init parse_early_options(char *cmdline)
{
	parse_args("early options", cmdline, NULL, 0, do_early_param);
}

L
Linus Torvalds 已提交
498 499 500 501 502 503 504 505 506 507
/* Arch code calls this early on, or if not, just before other parsing. */
void __init parse_early_param(void)
{
	static __initdata int done = 0;
	static __initdata char tmp_cmdline[COMMAND_LINE_SIZE];

	if (done)
		return;

	/* All fall through to do_early_param. */
508
	strlcpy(tmp_cmdline, boot_command_line, COMMAND_LINE_SIZE);
M
Magnus Damm 已提交
509
	parse_early_options(tmp_cmdline);
L
Linus Torvalds 已提交
510 511 512 513 514 515 516
	done = 1;
}

/*
 *	Activate the first processor.
 */

517 518 519 520
static void __init boot_cpu_init(void)
{
	int cpu = smp_processor_id();
	/* Mark the boot cpu "present", "online" etc for SMP and UP case */
521 522 523
	set_cpu_online(cpu, true);
	set_cpu_present(cpu, true);
	set_cpu_possible(cpu, true);
524 525
}

526
void __init __weak smp_setup_processor_id(void)
527 528 529
{
}

530 531 532 533
void __init __weak thread_info_cache_init(void)
{
}

P
Pekka Enberg 已提交
534 535 536 537 538
/*
 * Set up kernel memory allocators
 */
static void __init mm_init(void)
{
539 540 541 542 543
	/*
	 * page_cgroup requires countinous pages as memmap
	 * and it's bigger than MAX_ORDER unless SPARSEMEM.
	 */
	page_cgroup_init_flatmem();
P
Pekka Enberg 已提交
544 545
	mem_init();
	kmem_cache_init();
546
	pgtable_cache_init();
P
Pekka Enberg 已提交
547 548 549
	vmalloc_init();
}

L
Linus Torvalds 已提交
550 551 552 553
asmlinkage void __init start_kernel(void)
{
	char * command_line;
	extern struct kernel_param __start___param[], __stop___param[];
554 555 556

	smp_setup_processor_id();

I
Ingo Molnar 已提交
557 558 559 560 561
	/*
	 * Need to run as early as possible, to initialize the
	 * lockdep hash:
	 */
	lockdep_init();
562
	debug_objects_early_init();
563 564 565 566 567 568

	/*
	 * Set up the the initial canary ASAP:
	 */
	boot_init_stack_canary();

569
	cgroup_init_early();
I
Ingo Molnar 已提交
570 571 572

	local_irq_disable();
	early_boot_irqs_off();
I
Ingo Molnar 已提交
573
	early_init_irq_lock_class();
I
Ingo Molnar 已提交
574

L
Linus Torvalds 已提交
575 576 577 578 579
/*
 * Interrupts are still disabled. Do necessary setups, then
 * enable them
 */
	lock_kernel();
580
	tick_init();
581
	boot_cpu_init();
L
Linus Torvalds 已提交
582
	page_address_init();
A
Alex Riesen 已提交
583
	printk(KERN_NOTICE "%s", linux_banner);
L
Linus Torvalds 已提交
584
	setup_arch(&command_line);
585
	mm_init_owner(&init_mm, &init_task);
586
	setup_command_line(command_line);
587
	setup_nr_cpu_ids();
588
	setup_per_cpu_areas();
589
	smp_prepare_boot_cpu();	/* arch-specific boot-cpu hooks */
L
Linus Torvalds 已提交
590

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606
	build_all_zonelists();
	page_alloc_init();

	printk(KERN_NOTICE "Kernel command line: %s\n", boot_command_line);
	parse_early_param();
	parse_args("Booting kernel", static_command_line, __start___param,
		   __stop___param - __start___param,
		   &unknown_bootoption);
	/*
	 * These use large bootmem allocations and must precede
	 * kmem_cache_init()
	 */
	pidhash_init();
	vfs_caches_init_early();
	sort_main_extable();
	trap_init();
P
Pekka Enberg 已提交
607
	mm_init();
L
Linus Torvalds 已提交
608 609 610 611 612 613 614 615 616 617 618
	/*
	 * Set up the scheduler prior starting any interrupts (such as the
	 * timer interrupt). Full topology setup happens at smp_init()
	 * time - but meanwhile we still have a functioning scheduler.
	 */
	sched_init();
	/*
	 * Disable preemption - early bootup scheduling is extremely
	 * fragile until we cpu_idle() for the first time.
	 */
	preempt_disable();
619 620 621 622 623
	if (!irqs_disabled()) {
		printk(KERN_WARNING "start_kernel(): bug: interrupts were "
				"enabled *very* early, fixing it\n");
		local_irq_disable();
	}
L
Linus Torvalds 已提交
624
	rcu_init();
625 626
	/* init some links before init_ISA_irqs() */
	early_irq_init();
L
Linus Torvalds 已提交
627
	init_IRQ();
C
Catalin Marinas 已提交
628
	prio_tree_init();
L
Linus Torvalds 已提交
629
	init_timers();
630
	hrtimers_init();
L
Linus Torvalds 已提交
631
	softirq_init();
632
	timekeeping_init();
J
john stultz 已提交
633
	time_init();
634
	sched_clock_init();
635 636
	profile_init();
	if (!irqs_disabled())
637 638
		printk(KERN_CRIT "start_kernel(): bug: interrupts were "
				 "enabled early\n");
I
Ingo Molnar 已提交
639
	early_boot_irqs_on();
640
	local_irq_enable();
641 642 643 644

	/* Interrupts are enabled now so all GFP allocations are safe. */
	set_gfp_allowed_mask(__GFP_BITS_MASK);

645
	kmem_cache_init_late();
L
Linus Torvalds 已提交
646 647 648 649 650 651 652 653 654

	/*
	 * HACK ALERT! This is early. We're enabling the console before
	 * we've done PCI setups etc, and console_init() must be aware of
	 * this. But we do want output early, in case something goes wrong.
	 */
	console_init();
	if (panic_later)
		panic(panic_later, panic_param);
I
Ingo Molnar 已提交
655 656 657

	lockdep_info();

658 659 660 661 662 663 664
	/*
	 * Need to run this when irqs are enabled, because it wants
	 * to self-test [hard/soft]-irqs on/off lock inversion bugs
	 * too:
	 */
	locking_selftest();

L
Linus Torvalds 已提交
665 666
#ifdef CONFIG_BLK_DEV_INITRD
	if (initrd_start && !initrd_below_start_ok &&
667
	    page_to_pfn(virt_to_page((void *)initrd_start)) < min_low_pfn) {
L
Linus Torvalds 已提交
668
		printk(KERN_CRIT "initrd overwritten (0x%08lx < 0x%08lx) - "
669
		    "disabling it.\n",
670 671
		    page_to_pfn(virt_to_page((void *)initrd_start)),
		    min_low_pfn);
L
Linus Torvalds 已提交
672 673 674
		initrd_start = 0;
	}
#endif
675
	page_cgroup_init();
676
	enable_debug_pagealloc();
677
	kmemtrace_init();
C
Catalin Marinas 已提交
678
	kmemleak_init();
679
	debug_objects_mem_init();
680
	idr_init_cache();
681
	setup_per_cpu_pageset();
L
Linus Torvalds 已提交
682 683 684 685 686 687 688 689 690 691
	numa_policy_init();
	if (late_time_init)
		late_time_init();
	calibrate_delay();
	pidmap_init();
	anon_vma_init();
#ifdef CONFIG_X86
	if (efi_enabled)
		efi_enter_virtual_mode();
#endif
692
	thread_info_cache_init();
D
David Howells 已提交
693
	cred_init();
L
Linus Torvalds 已提交
694 695 696 697 698 699 700 701 702 703 704 705 706
	fork_init(num_physpages);
	proc_caches_init();
	buffer_init();
	key_init();
	security_init();
	vfs_caches_init(num_physpages);
	radix_tree_init();
	signals_init();
	/* rootfs populating might need page-writeback */
	page_writeback_init();
#ifdef CONFIG_PROC_FS
	proc_root_init();
#endif
707
	cgroup_init();
L
Linus Torvalds 已提交
708
	cpuset_init();
709
	taskstats_init_early();
710
	delayacct_init();
L
Linus Torvalds 已提交
711 712 713 714 715

	check_bugs();

	acpi_early_init(); /* before LAPIC and SMP init */

716 717
	ftrace_init();

L
Linus Torvalds 已提交
718 719 720 721
	/* Do the rest non-__init'ed, we're now alive */
	rest_init();
}

722 723 724 725 726 727 728 729 730 731 732
/* Call all constructor functions linked into the kernel. */
static void __init do_ctors(void)
{
#ifdef CONFIG_CONSTRUCTORS
	ctor_fn_t *call = (ctor_fn_t *) __ctors_start;

	for (; call < (ctor_fn_t *) __ctors_end; call++)
		(*call)();
#endif
}

733
int initcall_debug;
R
Rusty Russell 已提交
734
core_param(initcall_debug, initcall_debug, bool, 0644);
L
Linus Torvalds 已提交
735

736 737 738 739
static char msgbuf[64];
static struct boot_trace_call call;
static struct boot_trace_ret ret;

740
int do_one_initcall(initcall_t fn)
L
Linus Torvalds 已提交
741 742
{
	int count = preempt_count();
743
	ktime_t calltime, delta, rettime;
L
Linus Torvalds 已提交
744

745
	if (initcall_debug) {
746 747 748 749
		call.caller = task_pid_nr(current);
		printk("calling  %pF @ %i\n", fn, call.caller);
		calltime = ktime_get();
		trace_boot_call(&call, fn);
750
		enable_boot_trace();
751
	}
L
Linus Torvalds 已提交
752

753
	ret.result = fn();
L
Linus Torvalds 已提交
754

755
	if (initcall_debug) {
756
		disable_boot_trace();
757 758
		rettime = ktime_get();
		delta = ktime_sub(rettime, calltime);
759
		ret.duration = (unsigned long long) ktime_to_ns(delta) >> 10;
760
		trace_boot_ret(&ret, fn);
761
		printk("initcall %pF returned %d after %Ld usecs\n", fn,
762
			ret.result, ret.duration);
763
	}
764

765
	msgbuf[0] = 0;
766

767 768
	if (ret.result && ret.result != -ENODEV && initcall_debug)
		sprintf(msgbuf, "error code %d ", ret.result);
769

770
	if (preempt_count() != count) {
771
		strlcat(msgbuf, "preemption imbalance ", sizeof(msgbuf));
772
		preempt_count() = count;
L
Linus Torvalds 已提交
773
	}
774
	if (irqs_disabled()) {
775
		strlcat(msgbuf, "disabled interrupts ", sizeof(msgbuf));
776 777 778
		local_irq_enable();
	}
	if (msgbuf[0]) {
779
		printk("initcall %pF returned with %s\n", fn, msgbuf);
780
	}
781

782
	return ret.result;
783 784 785
}


786
extern initcall_t __initcall_start[], __initcall_end[], __early_initcall_end[];
787 788 789 790 791

static void __init do_initcalls(void)
{
	initcall_t *call;

792
	for (call = __early_initcall_end; call < __initcall_end; call++)
793
		do_one_initcall(*call);
L
Linus Torvalds 已提交
794 795 796 797 798 799 800 801 802 803 804 805 806 807

	/* Make sure there is no pending stuff from the initcall sequence */
	flush_scheduled_work();
}

/*
 * Ok, the machine is now initialized. None of the devices
 * have been touched yet, but the CPU subsystem is up and
 * running, and memory and process management works.
 *
 * Now we can finally start doing some real work..
 */
static void __init do_basic_setup(void)
{
P
Paul E. McKenney 已提交
808
	rcu_init_sched(); /* needed by module_init stage. */
809
	init_workqueues();
810
	cpuset_init_smp();
L
Linus Torvalds 已提交
811 812
	usermodehelper_init();
	driver_init();
813
	init_irq_proc();
814
	do_ctors();
L
Linus Torvalds 已提交
815 816 817
	do_initcalls();
}

818
static void __init do_pre_smp_initcalls(void)
819 820 821 822 823 824 825
{
	initcall_t *call;

	for (call = __initcall_start; call < __early_initcall_end; call++)
		do_one_initcall(*call);
}

L
Linus Torvalds 已提交
826 827 828
static void run_init_process(char *init_filename)
{
	argv_init[0] = init_filename;
A
Arnd Bergmann 已提交
829
	kernel_execve(init_filename, argv_init, envp_init);
L
Linus Torvalds 已提交
830 831
}

832 833 834
/* This is a non __init function. Force it to be noinline otherwise gcc
 * makes it inline to init() and it becomes part of init.text section
 */
835
static noinline int init_post(void)
836
	__releases(kernel_lock)
837
{
838 839
	/* need to finish all async __init code before freeing the memory */
	async_synchronize_full();
840 841 842 843 844 845 846 847 848 849 850 851
	free_initmem();
	unlock_kernel();
	mark_rodata_ro();
	system_state = SYSTEM_RUNNING;
	numa_default_policy();

	if (sys_open((const char __user *) "/dev/console", O_RDWR, 0) < 0)
		printk(KERN_WARNING "Warning: unable to open an initial console.\n");

	(void) sys_dup(0);
	(void) sys_dup(0);

852 853
	current->signal->flags |= SIGNAL_UNKILLABLE;

854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878
	if (ramdisk_execute_command) {
		run_init_process(ramdisk_execute_command);
		printk(KERN_WARNING "Failed to execute %s\n",
				ramdisk_execute_command);
	}

	/*
	 * We try each of these until one succeeds.
	 *
	 * The Bourne shell can be used instead of init if we are
	 * trying to recover a really broken machine.
	 */
	if (execute_command) {
		run_init_process(execute_command);
		printk(KERN_WARNING "Failed to execute %s.  Attempting "
					"defaults...\n", execute_command);
	}
	run_init_process("/sbin/init");
	run_init_process("/etc/init");
	run_init_process("/bin/init");
	run_init_process("/bin/sh");

	panic("No init found.  Try passing init= option to kernel.");
}

879
static int __init kernel_init(void * unused)
L
Linus Torvalds 已提交
880 881
{
	lock_kernel();
882 883 884 885 886

	/*
	 * init can allocate pages on any node
	 */
	set_mems_allowed(node_possible_map);
L
Linus Torvalds 已提交
887 888 889
	/*
	 * init can run on any cpu.
	 */
890
	set_cpus_allowed_ptr(current, cpu_all_mask);
L
Linus Torvalds 已提交
891 892 893 894 895 896 897 898
	/*
	 * Tell the world that we're going to be the grim
	 * reaper of innocent orphaned children.
	 *
	 * We don't want people to have to make incorrect
	 * assumptions about where in the task array this
	 * can be found.
	 */
899
	init_pid_ns.child_reaper = current;
L
Linus Torvalds 已提交
900

901 902
	cad_pid = task_pid(current);

903
	smp_prepare_cpus(setup_max_cpus);
L
Linus Torvalds 已提交
904 905

	do_pre_smp_initcalls();
906
	start_boot_trace();
L
Linus Torvalds 已提交
907 908 909 910 911 912 913 914 915 916

	smp_init();
	sched_init_smp();

	do_basic_setup();

	/*
	 * check if there is an early userspace init.  If yes, let it do all
	 * the work
	 */
917 918 919 920 921 922

	if (!ramdisk_execute_command)
		ramdisk_execute_command = "/init";

	if (sys_access((const char __user *) ramdisk_execute_command, 0) != 0) {
		ramdisk_execute_command = NULL;
L
Linus Torvalds 已提交
923
		prepare_namespace();
924
	}
L
Linus Torvalds 已提交
925 926 927 928 929 930

	/*
	 * Ok, we have completed the initial bootup, and
	 * we're essentially up and running. Get rid of the
	 * initmem segments and start the user-mode stuff..
	 */
931

932 933
	init_post();
	return 0;
L
Linus Torvalds 已提交
934
}