vdso32-setup.c 9.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * (C) Copyright 2002 Linus Torvalds
3 4
 * Portions based on the vdso-randomization code from exec-shield:
 * Copyright(C) 2005-2006, Red Hat, Inc., Ingo Molnar
L
Linus Torvalds 已提交
5 6 7 8 9 10 11 12 13 14 15
 *
 * This file contains the needed initializations to support sysenter.
 */

#include <linux/init.h>
#include <linux/smp.h>
#include <linux/thread_info.h>
#include <linux/sched.h>
#include <linux/gfp.h>
#include <linux/string.h>
#include <linux/elf.h>
16
#include <linux/mm.h>
A
Alexey Dobriyan 已提交
17
#include <linux/err.h>
18
#include <linux/module.h>
L
Linus Torvalds 已提交
19 20 21 22 23

#include <asm/cpufeature.h>
#include <asm/msr.h>
#include <asm/pgtable.h>
#include <asm/unistd.h>
24
#include <asm/elf.h>
25
#include <asm/tlbflush.h>
R
Roland McGrath 已提交
26
#include <asm/vdso.h>
R
Roland McGrath 已提交
27
#include <asm/proto.h>
28 29 30 31 32 33 34 35 36 37 38 39

enum {
	VDSO_DISABLED = 0,
	VDSO_ENABLED = 1,
	VDSO_COMPAT = 2,
};

#ifdef CONFIG_COMPAT_VDSO
#define VDSO_DEFAULT	VDSO_COMPAT
#else
#define VDSO_DEFAULT	VDSO_ENABLED
#endif
L
Linus Torvalds 已提交
40

R
Roland McGrath 已提交
41 42 43 44 45 46 47 48 49 50 51 52
#ifdef CONFIG_X86_64
#define vdso_enabled			sysctl_vsyscall32
#define arch_setup_additional_pages	syscall32_setup_pages
#endif

/*
 * This is the difference between the prelinked addresses in the vDSO images
 * and the VDSO_HIGH_BASE address where CONFIG_COMPAT_VDSO places the vDSO
 * in the user address space.
 */
#define VDSO_ADDR_ADJUST	(VDSO_HIGH_BASE - (unsigned long)VDSO32_PRELINK)

53 54 55 56
/*
 * Should the kernel map a VDSO page into processes and pass its
 * address down to glibc upon exec()?
 */
57
unsigned int __read_mostly vdso_enabled = VDSO_DEFAULT;
58 59 60 61 62 63 64 65

static int __init vdso_setup(char *s)
{
	vdso_enabled = simple_strtoul(s, NULL, 0);

	return 1;
}

R
Roland McGrath 已提交
66 67 68 69 70 71
/*
 * For consistency, the argument vdso32=[012] affects the 32-bit vDSO
 * behavior on both 64-bit and 32-bit kernels.
 * On 32-bit kernels, vdso=[012] means the same thing.
 */
__setup("vdso32=", vdso_setup);
72

R
Roland McGrath 已提交
73 74 75 76 77
#ifdef CONFIG_X86_32
__setup_param("vdso=", vdso32_setup, vdso_setup, 0);

EXPORT_SYMBOL_GPL(vdso_enabled);
#endif
L
Linus Torvalds 已提交
78

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
static __init void reloc_symtab(Elf32_Ehdr *ehdr,
				unsigned offset, unsigned size)
{
	Elf32_Sym *sym = (void *)ehdr + offset;
	unsigned nsym = size / sizeof(*sym);
	unsigned i;

	for(i = 0; i < nsym; i++, sym++) {
		if (sym->st_shndx == SHN_UNDEF ||
		    sym->st_shndx == SHN_ABS)
			continue;  /* skip */

		if (sym->st_shndx > SHN_LORESERVE) {
			printk(KERN_INFO "VDSO: unexpected st_shndx %x\n",
			       sym->st_shndx);
			continue;
		}

		switch(ELF_ST_TYPE(sym->st_info)) {
		case STT_OBJECT:
		case STT_FUNC:
		case STT_SECTION:
		case STT_FILE:
R
Roland McGrath 已提交
102
			sym->st_value += VDSO_ADDR_ADJUST;
103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
		}
	}
}

static __init void reloc_dyn(Elf32_Ehdr *ehdr, unsigned offset)
{
	Elf32_Dyn *dyn = (void *)ehdr + offset;

	for(; dyn->d_tag != DT_NULL; dyn++)
		switch(dyn->d_tag) {
		case DT_PLTGOT:
		case DT_HASH:
		case DT_STRTAB:
		case DT_SYMTAB:
		case DT_RELA:
		case DT_INIT:
		case DT_FINI:
		case DT_REL:
		case DT_DEBUG:
		case DT_JMPREL:
		case DT_VERSYM:
		case DT_VERDEF:
		case DT_VERNEED:
		case DT_ADDRRNGLO ... DT_ADDRRNGHI:
			/* definitely pointers needing relocation */
R
Roland McGrath 已提交
128
			dyn->d_un.d_ptr += VDSO_ADDR_ADJUST;
129 130 131 132 133 134 135 136
			break;

		case DT_ENCODING ... OLD_DT_LOOS-1:
		case DT_LOOS ... DT_HIOS-1:
			/* Tags above DT_ENCODING are pointers if
			   they're even */
			if (dyn->d_tag >= DT_ENCODING &&
			    (dyn->d_tag & 1) == 0)
R
Roland McGrath 已提交
137
				dyn->d_un.d_ptr += VDSO_ADDR_ADJUST;
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
			break;

		case DT_VERDEFNUM:
		case DT_VERNEEDNUM:
		case DT_FLAGS_1:
		case DT_RELACOUNT:
		case DT_RELCOUNT:
		case DT_VALRNGLO ... DT_VALRNGHI:
			/* definitely not pointers */
			break;

		case OLD_DT_LOOS ... DT_LOOS-1:
		case DT_HIOS ... DT_VALRNGLO-1:
		default:
			if (dyn->d_tag > DT_ENCODING)
				printk(KERN_INFO "VDSO: unexpected DT_tag %x\n",
				       dyn->d_tag);
			break;
		}
}

static __init void relocate_vdso(Elf32_Ehdr *ehdr)
{
	Elf32_Phdr *phdr;
	Elf32_Shdr *shdr;
	int i;

165
	BUG_ON(memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0 ||
R
Roland McGrath 已提交
166
	       !elf_check_arch_ia32(ehdr) ||
167 168
	       ehdr->e_type != ET_DYN);

R
Roland McGrath 已提交
169
	ehdr->e_entry += VDSO_ADDR_ADJUST;
170 171 172 173

	/* rebase phdrs */
	phdr = (void *)ehdr + ehdr->e_phoff;
	for (i = 0; i < ehdr->e_phnum; i++) {
R
Roland McGrath 已提交
174
		phdr[i].p_vaddr += VDSO_ADDR_ADJUST;
175 176 177 178 179 180 181 182 183 184 185 186

		/* relocate dynamic stuff */
		if (phdr[i].p_type == PT_DYNAMIC)
			reloc_dyn(ehdr, phdr[i].p_offset);
	}

	/* rebase sections */
	shdr = (void *)ehdr + ehdr->e_shoff;
	for(i = 0; i < ehdr->e_shnum; i++) {
		if (!(shdr[i].sh_flags & SHF_ALLOC))
			continue;

R
Roland McGrath 已提交
187
		shdr[i].sh_addr += VDSO_ADDR_ADJUST;
188 189 190 191 192 193 194 195

		if (shdr[i].sh_type == SHT_SYMTAB ||
		    shdr[i].sh_type == SHT_DYNSYM)
			reloc_symtab(ehdr, shdr[i].sh_offset,
				     shdr[i].sh_size);
	}
}

R
Roland McGrath 已提交
196 197 198 199
static struct page *vdso32_pages[1];

#ifdef CONFIG_X86_64

200
#define	vdso32_sysenter()	(boot_cpu_has(X86_FEATURE_SYSENTER32))
201
#define	vdso32_syscall()	(boot_cpu_has(X86_FEATURE_SYSCALL32))
R
Roland McGrath 已提交
202 203 204 205 206 207

/* May not be __init: called during resume */
void syscall32_cpu_init(void)
{
	/* Load these always in case some future AMD CPU supports
	   SYSENTER from compat mode too. */
208 209 210
	wrmsrl_safe(MSR_IA32_SYSENTER_CS, (u64)__KERNEL_CS);
	wrmsrl_safe(MSR_IA32_SYSENTER_ESP, 0ULL);
	wrmsrl_safe(MSR_IA32_SYSENTER_EIP, (u64)ia32_sysenter_target);
R
Roland McGrath 已提交
211 212 213 214 215 216 217 218 219 220 221 222 223

	wrmsrl(MSR_CSTAR, ia32_cstar_target);
}

#define compat_uses_vma		1

static inline void map_compat_vdso(int map)
{
}

#else  /* CONFIG_X86_32 */

#define vdso32_sysenter()	(boot_cpu_has(X86_FEATURE_SEP))
224
#define vdso32_syscall()	(0)
R
Roland McGrath 已提交
225

L
Li Shaohua 已提交
226
void enable_sep_cpu(void)
L
Linus Torvalds 已提交
227 228 229 230
{
	int cpu = get_cpu();
	struct tss_struct *tss = &per_cpu(init_tss, cpu);

L
Li Shaohua 已提交
231 232 233 234 235
	if (!boot_cpu_has(X86_FEATURE_SEP)) {
		put_cpu();
		return;
	}

236
	tss->x86_tss.ss1 = __KERNEL_CS;
237
	tss->x86_tss.sp1 = sizeof(struct tss_struct) + (unsigned long) tss;
L
Linus Torvalds 已提交
238
	wrmsr(MSR_IA32_SYSENTER_CS, __KERNEL_CS, 0);
239
	wrmsr(MSR_IA32_SYSENTER_ESP, tss->x86_tss.sp1, 0);
R
Roland McGrath 已提交
240
	wrmsr(MSR_IA32_SYSENTER_EIP, (unsigned long) ia32_sysenter_target, 0);
L
Linus Torvalds 已提交
241 242 243
	put_cpu();	
}

244 245 246 247 248 249 250 251 252
static struct vm_area_struct gate_vma;

static int __init gate_vma_init(void)
{
	gate_vma.vm_mm = NULL;
	gate_vma.vm_start = FIXADDR_USER_START;
	gate_vma.vm_end = FIXADDR_USER_END;
	gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
	gate_vma.vm_page_prot = __P101;
253

254 255 256
	return 0;
}

R
Roland McGrath 已提交
257
#define compat_uses_vma		0
L
Linus Torvalds 已提交
258

259 260 261 262 263 264 265 266 267
static void map_compat_vdso(int map)
{
	static int vdso_mapped;

	if (map == vdso_mapped)
		return;

	vdso_mapped = map;

R
Roland McGrath 已提交
268
	__set_fixmap(FIX_VDSO, page_to_pfn(vdso32_pages[0]) << PAGE_SHIFT,
269 270 271 272 273 274
		     map ? PAGE_READONLY_EXEC : PAGE_NONE);

	/* flush stray tlbs */
	flush_tlb_all();
}

R
Roland McGrath 已提交
275 276
#endif	/* CONFIG_X86_64 */

277
int __init sysenter_setup(void)
L
Linus Torvalds 已提交
278
{
279
	void *syscall_page = (void *)get_zeroed_page(GFP_ATOMIC);
280 281 282
	const void *vsyscall;
	size_t vsyscall_len;

R
Roland McGrath 已提交
283
	vdso32_pages[0] = virt_to_page(syscall_page);
L
Linus Torvalds 已提交
284

R
Roland McGrath 已提交
285
#ifdef CONFIG_X86_32
286
	gate_vma_init();
R
Roland McGrath 已提交
287
#endif
L
Linus Torvalds 已提交
288

289 290 291 292
	if (vdso32_syscall()) {
		vsyscall = &vdso32_syscall_start;
		vsyscall_len = &vdso32_syscall_end - &vdso32_syscall_start;
	} else if (vdso32_sysenter()){
R
Roland McGrath 已提交
293 294
		vsyscall = &vdso32_sysenter_start;
		vsyscall_len = &vdso32_sysenter_end - &vdso32_sysenter_start;
295 296 297
	} else {
		vsyscall = &vdso32_int80_start;
		vsyscall_len = &vdso32_int80_end - &vdso32_int80_start;
L
Linus Torvalds 已提交
298 299
	}

300 301
	memcpy(syscall_page, vsyscall, vsyscall_len);
	relocate_vdso(syscall_page);
L
Linus Torvalds 已提交
302 303 304

	return 0;
}
305 306

/* Setup a VMA at program startup for the vsyscall page */
307
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
308 309 310
{
	struct mm_struct *mm = current->mm;
	unsigned long addr;
311
	int ret = 0;
312
	bool compat;
313

H
H. J. Lu 已提交
314 315
#ifdef CONFIG_X86_X32_ABI
	if (test_thread_flag(TIF_X32))
316
		return x32_setup_additional_pages(bprm, uses_interp);
H
H. J. Lu 已提交
317 318
#endif

319 320 321
	if (vdso_enabled == VDSO_DISABLED)
		return 0;

322 323
	down_write(&mm->mmap_sem);

324 325 326 327 328 329 330 331 332 333 334 335 336 337
	/* Test compat mode once here, in case someone
	   changes it via sysctl */
	compat = (vdso_enabled == VDSO_COMPAT);

	map_compat_vdso(compat);

	if (compat)
		addr = VDSO_HIGH_BASE;
	else {
		addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0);
		if (IS_ERR_VALUE(addr)) {
			ret = addr;
			goto up_fail;
		}
R
Roland McGrath 已提交
338
	}
339

340 341
	current->mm->context.vdso = (void *)addr;

R
Roland McGrath 已提交
342
	if (compat_uses_vma || !compat) {
343 344 345 346 347
		/*
		 * MAYWRITE to allow gdb to COW and set breakpoints
		 */
		ret = install_special_mapping(mm, addr, PAGE_SIZE,
					      VM_READ|VM_EXEC|
348
					      VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
R
Roland McGrath 已提交
349
					      vdso32_pages);
350 351 352 353

		if (ret)
			goto up_fail;
	}
354 355

	current_thread_info()->sysenter_return =
R
Roland McGrath 已提交
356
		VDSO32_SYMBOL(addr, SYSENTER_RETURN);
357 358

  up_fail:
359 360 361
	if (ret)
		current->mm->context.vdso = NULL;

362
	up_write(&mm->mmap_sem);
363

364 365 366
	return ret;
}

R
Roland McGrath 已提交
367 368
#ifdef CONFIG_X86_64

369
subsys_initcall(sysenter_setup);
R
Roland McGrath 已提交
370

R
Roland McGrath 已提交
371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402
#ifdef CONFIG_SYSCTL
/* Register vsyscall32 into the ABI table */
#include <linux/sysctl.h>

static ctl_table abi_table2[] = {
	{
		.procname	= "vsyscall32",
		.data		= &sysctl_vsyscall32,
		.maxlen		= sizeof(int),
		.mode		= 0644,
		.proc_handler	= proc_dointvec
	},
	{}
};

static ctl_table abi_root_table2[] = {
	{
		.procname = "abi",
		.mode = 0555,
		.child = abi_table2
	},
	{}
};

static __init int ia32_binfmt_init(void)
{
	register_sysctl_table(abi_root_table2);
	return 0;
}
__initcall(ia32_binfmt_init);
#endif

R
Roland McGrath 已提交
403 404
#else  /* CONFIG_X86_32 */

405 406 407 408 409 410 411
const char *arch_vma_name(struct vm_area_struct *vma)
{
	if (vma->vm_mm && vma->vm_start == (long)vma->vm_mm->context.vdso)
		return "[vdso]";
	return NULL;
}

412
struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
413
{
414 415 416 417
	/*
	 * Check to see if the corresponding task was created in compat vdso
	 * mode.
	 */
418 419
	if (mm && mm->context.vdso == (void *)VDSO_HIGH_BASE)
		return &gate_vma;
420 421 422
	return NULL;
}

423
int in_gate_area(struct mm_struct *mm, unsigned long addr)
424
{
425
	const struct vm_area_struct *vma = get_gate_vma(mm);
426 427

	return vma && addr >= vma->vm_start && addr < vma->vm_end;
428 429
}

430
int in_gate_area_no_mm(unsigned long addr)
431 432 433
{
	return 0;
}
R
Roland McGrath 已提交
434 435

#endif	/* CONFIG_X86_64 */