diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S index 449e76f118d307eeab04842af47514272cbe0873..6f4f0378e7089302eaba169d94282488afb54db5 100644 --- a/arch/alpha/kernel/vmlinux.lds.S +++ b/arch/alpha/kernel/vmlinux.lds.S @@ -69,10 +69,7 @@ SECTIONS . = ALIGN(8); SECURITY_INIT - . = ALIGN(8192); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(8192) . = ALIGN(2*8192); __init_end = .; diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S index 2b7a8f5d8cf276fe1e31ec38cf4adcb909879cb2..5ff5406666b437c9a8a5b848279d60735b601bc6 100644 --- a/arch/arm/kernel/vmlinux.lds.S +++ b/arch/arm/kernel/vmlinux.lds.S @@ -66,6 +66,7 @@ SECTIONS . = ALIGN(4096); __per_cpu_start = .; *(.data.percpu) + *(.data.percpu.shared_aligned) __per_cpu_end = .; #ifndef CONFIG_XIP_KERNEL __init_begin = _stext; diff --git a/arch/cris/arch-v32/vmlinux.lds.S b/arch/cris/arch-v32/vmlinux.lds.S index dfa25e1542b91dec2a52a5f044c45589956c702f..651a77f2ccc4f62b26f7fa070e990c4580c5727b 100644 --- a/arch/cris/arch-v32/vmlinux.lds.S +++ b/arch/cris/arch-v32/vmlinux.lds.S @@ -91,10 +91,7 @@ SECTIONS } SECURITY_INIT - . = ALIGN (8192); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(8192) #ifdef CONFIG_BLK_DEV_INITRD .init.ramfs : { diff --git a/arch/frv/kernel/vmlinux.lds.S b/arch/frv/kernel/vmlinux.lds.S index 481dc137464016a6e37adc46ed82ba0139a32861..3b71e0c863996f66828dea23286c616e695176c8 100644 --- a/arch/frv/kernel/vmlinux.lds.S +++ b/arch/frv/kernel/vmlinux.lds.S @@ -57,10 +57,7 @@ SECTIONS __alt_instructions_end = .; .altinstr_replacement : { *(.altinstr_replacement) } - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) #ifdef CONFIG_BLK_DEV_INITRD . = ALIGN(4096); diff --git a/arch/i386/kernel/vmlinux.lds.S b/arch/i386/kernel/vmlinux.lds.S index 00f1bc47d3a248813f15519ac0175e43f4d904cd..4dc44b8007ccbe9ea8c44f5e96e33f50b2da4b50 100644 --- a/arch/i386/kernel/vmlinux.lds.S +++ b/arch/i386/kernel/vmlinux.lds.S @@ -181,6 +181,7 @@ SECTIONS .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { __per_cpu_start = .; *(.data.percpu) + *(.data.percpu.shared_aligned) __per_cpu_end = .; } . = ALIGN(4096); diff --git a/arch/ia64/kernel/vmlinux.lds.S b/arch/ia64/kernel/vmlinux.lds.S index 5a65965c8b53b4c3e787b5f17025af610dce4626..860f251d2fc26f7188551b627ef76c42e5f7f1bb 100644 --- a/arch/ia64/kernel/vmlinux.lds.S +++ b/arch/ia64/kernel/vmlinux.lds.S @@ -206,6 +206,7 @@ SECTIONS { __per_cpu_start = .; *(.data.percpu) + *(.data.percpu.shared_aligned) __per_cpu_end = .; } . = __phys_per_cpu_start + PERCPU_PAGE_SIZE; /* ensure percpu data fits diff --git a/arch/m32r/kernel/vmlinux.lds.S b/arch/m32r/kernel/vmlinux.lds.S index 4e2d5b9f0a9abc601e591040bc1c3a5517d3a0b8..942a8c7a44174b0c010d88c10694523ec93c248f 100644 --- a/arch/m32r/kernel/vmlinux.lds.S +++ b/arch/m32r/kernel/vmlinux.lds.S @@ -110,10 +110,7 @@ SECTIONS __initramfs_end = .; #endif - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) . = ALIGN(4096); __init_end = .; /* freed after init ends here */ diff --git a/arch/mips/kernel/vmlinux.lds.S b/arch/mips/kernel/vmlinux.lds.S index 9b9992cd562acd17ee03cc94d69be6acfdfbad1e..bc9bae2a73f4843da0d38cb8f17c4b05b8d81e5c 100644 --- a/arch/mips/kernel/vmlinux.lds.S +++ b/arch/mips/kernel/vmlinux.lds.S @@ -119,10 +119,7 @@ SECTIONS .init.ramfs : { *(.init.ramfs) } __initramfs_end = .; #endif - . = ALIGN(_PAGE_SIZE); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(_PAGE_SIZE) . = ALIGN(_PAGE_SIZE); __init_end = .; /* freed after init ends here */ diff --git a/arch/parisc/kernel/vmlinux.lds.S b/arch/parisc/kernel/vmlinux.lds.S index 4d96ba4b984986ce4c018a070b46ef72c2ea00a3..d4e6a93c8d9a60df2862b5dbe66b5fa87c7f9280 100644 --- a/arch/parisc/kernel/vmlinux.lds.S +++ b/arch/parisc/kernel/vmlinux.lds.S @@ -181,10 +181,9 @@ SECTIONS .init.ramfs : { *(.init.ramfs) } __initramfs_end = .; #endif - . = ALIGN(ASM_PAGE_SIZE); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + + PERCPU(ASM_PAGE_SIZE) + . = ALIGN(ASM_PAGE_SIZE); __init_end = .; /* freed after init ends here */ diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index ae4acd84143d91859a418e20229bfcb32589a28e..39fda6e6aa47313003c1ac407110630a136a97ae 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -144,6 +144,7 @@ SECTIONS .data.percpu : { __per_cpu_start = .; *(.data.percpu) + *(.data.percpu.shared_aligned) __per_cpu_end = .; } diff --git a/arch/ppc/kernel/vmlinux.lds.S b/arch/ppc/kernel/vmlinux.lds.S index 19db8746ff1460a6fcc716bf5db2219e5e09de1d..c0aac3ff9e91443672e54dfbcf1419073de60b33 100644 --- a/arch/ppc/kernel/vmlinux.lds.S +++ b/arch/ppc/kernel/vmlinux.lds.S @@ -130,10 +130,7 @@ SECTIONS __ftr_fixup : { *(__ftr_fixup) } __stop___ftr_fixup = .; - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) #ifdef CONFIG_BLK_DEV_INITRD . = ALIGN(4096); diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 7158a804a5e4f4ca0d4ff00d3773c8bf5474eff9..61ffd50fac40866470b59d12bfbb2c4f39571918 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -107,10 +107,7 @@ SECTIONS . = ALIGN(2); __initramfs_end = .; #endif - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) . = ALIGN(4096); __init_end = .; /* freed after init ends here */ diff --git a/arch/sh/kernel/vmlinux.lds.S b/arch/sh/kernel/vmlinux.lds.S index 0696402f446a28d5236eab374632fec28ccff4bb..5ba216180b304132f51ccf94a02c02bf5387dbd2 100644 --- a/arch/sh/kernel/vmlinux.lds.S +++ b/arch/sh/kernel/vmlinux.lds.S @@ -60,10 +60,7 @@ SECTIONS . = ALIGN(PAGE_SIZE); __nosave_end = .; - . = ALIGN(PAGE_SIZE); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(PAGE_SIZE) .data.cacheline_aligned : { *(.data.cacheline_aligned) } _edata = .; /* End of data section */ diff --git a/arch/sh64/kernel/vmlinux.lds.S b/arch/sh64/kernel/vmlinux.lds.S index 02aea86c5907cb5de7c8480b4ce892915a52d40a..8ac9c7c5f8487fd31c872a5ebb1ec7e354564b1a 100644 --- a/arch/sh64/kernel/vmlinux.lds.S +++ b/arch/sh64/kernel/vmlinux.lds.S @@ -87,7 +87,10 @@ SECTIONS . = ALIGN(PAGE_SIZE); __per_cpu_start = .; - .data.percpu : C_PHYS(.data.percpu) { *(.data.percpu) } + .data.percpu : C_PHYS(.data.percpu) { + *(.data.percpu) + *(.data.percpu.shared_aligned) + } __per_cpu_end = . ; .data.cacheline_aligned : C_PHYS(.data.cacheline_aligned) { *(.data.cacheline_aligned) } diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index f75a1b822789ec046beb013f2f6720b94375f8ec..47583887abc6fc823c2b96fd5cf43cf75a6455c1 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -65,10 +65,7 @@ SECTIONS __initramfs_end = .; #endif - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) . = ALIGN(4096); __init_end = .; . = ALIGN(32); diff --git a/arch/sparc64/kernel/vmlinux.lds.S b/arch/sparc64/kernel/vmlinux.lds.S index 3ad10f3027e4b2e82790dd826db811bf136b72f8..481861764deb16f4805522090f3f9406d2b4ad4e 100644 --- a/arch/sparc64/kernel/vmlinux.lds.S +++ b/arch/sparc64/kernel/vmlinux.lds.S @@ -90,10 +90,8 @@ SECTIONS __initramfs_end = .; #endif - . = ALIGN(PAGE_SIZE); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(PAGE_SIZE) + . = ALIGN(PAGE_SIZE); __init_end = .; __bss_start = .; diff --git a/arch/x86_64/kernel/vmlinux.lds.S b/arch/x86_64/kernel/vmlinux.lds.S index dbccfda8364f913c12167c25da77d5573eff364d..22590690a8a014b314538b5e6c0ae6eb7c8d7693 100644 --- a/arch/x86_64/kernel/vmlinux.lds.S +++ b/arch/x86_64/kernel/vmlinux.lds.S @@ -194,10 +194,8 @@ SECTIONS __initramfs_end = .; #endif - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) + . = ALIGN(4096); __init_end = .; diff --git a/arch/xtensa/kernel/vmlinux.lds.S b/arch/xtensa/kernel/vmlinux.lds.S index b0582c3c5f8da9dad2f43eb5b9d81ca9a14cbfb0..3e31512109f9c198c355f9497dc11ad54fdd2c05 100644 --- a/arch/xtensa/kernel/vmlinux.lds.S +++ b/arch/xtensa/kernel/vmlinux.lds.S @@ -190,10 +190,7 @@ SECTIONS __initramfs_end = .; #endif - . = ALIGN(4096); - __per_cpu_start = .; - .data.percpu : { *(.data.percpu) } - __per_cpu_end = .; + PERCPU(4096) /* We need this dummy segment here */ diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index d984a90414361b8f087cfcc83e8f15b3dcbe2989..d85172e9ed4580afa55c333e49fcb2ee925403db 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -14,6 +14,11 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp + /* var is in discarded region: offset to particular copy we want */ #define per_cpu(var, cpu) (*({ \ extern int simple_identifier_##var(void); \ @@ -34,6 +39,9 @@ do { \ #define DEFINE_PER_CPU(type, name) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) + #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var)) #define __get_cpu_var(var) per_cpu__##var #define __raw_get_cpu_var(var) per_cpu__##var diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 84155eb67f1d3c7907c4453548e683c2c4ea8952..a2b09ed852ad9e8ac09e2f7fc9be6d30366ea6f7 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -245,3 +245,11 @@ *(.initcall7.init) \ *(.initcall7s.init) +#define PERCPU(align) \ + . = ALIGN(align); \ + __per_cpu_start = .; \ + .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \ + *(.data.percpu) \ + *(.data.percpu.shared_aligned) \ + } \ + __per_cpu_end = .; diff --git a/include/asm-i386/percpu.h b/include/asm-i386/percpu.h index f54830b5d5ac299c38c496e65ed17d699c8c895f..a7ebd436f3cc513f61f6d154ee59566cd0de9fd3 100644 --- a/include/asm-i386/percpu.h +++ b/include/asm-i386/percpu.h @@ -54,6 +54,11 @@ extern unsigned long __per_cpu_offset[]; #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp + /* We can use this directly for local CPU (faster). */ DECLARE_PER_CPU(unsigned long, this_cpu_off); diff --git a/include/asm-ia64/percpu.h b/include/asm-ia64/percpu.h index fbe5cf3ab8dc0db74001a074960e8217b92bcb06..43a7aac414e04e7d74a1a6b45065b1a3b3c3846e 100644 --- a/include/asm-ia64/percpu.h +++ b/include/asm-ia64/percpu.h @@ -29,6 +29,16 @@ __attribute__((__section__(".data.percpu"))) \ __SMALL_ADDR_AREA __typeof__(type) per_cpu__##name +#ifdef CONFIG_SMP +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __SMALL_ADDR_AREA __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp +#else +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) +#endif + /* * Pretty much a literal copy of asm-generic/percpu.h, except that percpu_modcopy() is an * external routine, to avoid include-hell. diff --git a/include/asm-powerpc/percpu.h b/include/asm-powerpc/percpu.h index 2f2e3024fa619e23bb8d9dd4d47fc60c8551bd19..73dc8ba4010da31ce1afbdf4f0c90082bfec56c1 100644 --- a/include/asm-powerpc/percpu.h +++ b/include/asm-powerpc/percpu.h @@ -20,6 +20,11 @@ #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp + /* var is in discarded region: offset to particular copy we want */ #define per_cpu(var, cpu) (*RELOC_HIDE(&per_cpu__##var, __per_cpu_offset(cpu))) #define __get_cpu_var(var) (*RELOC_HIDE(&per_cpu__##var, __my_cpu_offset())) @@ -40,6 +45,8 @@ extern void setup_per_cpu_areas(void); #define DEFINE_PER_CPU(type, name) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var)) #define __get_cpu_var(var) per_cpu__##var diff --git a/include/asm-s390/percpu.h b/include/asm-s390/percpu.h index 9ea7f1023e578c0ff0a4eff47e77c5028d7e171b..545857e6444376352344db9c3f328d5125847105 100644 --- a/include/asm-s390/percpu.h +++ b/include/asm-s390/percpu.h @@ -41,6 +41,11 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; __attribute__((__section__(".data.percpu"))) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp + #define __get_cpu_var(var) __reloc_hide(var,S390_lowcore.percpu_offset) #define __raw_get_cpu_var(var) __reloc_hide(var,S390_lowcore.percpu_offset) #define per_cpu(var,cpu) __reloc_hide(var,__per_cpu_offset[cpu]) @@ -59,6 +64,8 @@ do { \ #define DEFINE_PER_CPU(type, name) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) #define __get_cpu_var(var) __reloc_hide(var,0) #define __raw_get_cpu_var(var) __reloc_hide(var,0) diff --git a/include/asm-sparc64/percpu.h b/include/asm-sparc64/percpu.h index 88db872ce2f8aed9c2b23689dc04375976e0d363..caf8750792ff16e2cad134a3969f9cafb6455a6b 100644 --- a/include/asm-sparc64/percpu.h +++ b/include/asm-sparc64/percpu.h @@ -18,6 +18,11 @@ extern unsigned long __per_cpu_shift; #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_aligned_in_smp + register unsigned long __local_per_cpu_offset asm("g5"); /* var is in discarded region: offset to particular copy we want */ @@ -38,6 +43,8 @@ do { \ #define real_setup_per_cpu_areas() do { } while (0) #define DEFINE_PER_CPU(type, name) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) #define per_cpu(var, cpu) (*((void)cpu, &per_cpu__##var)) #define __get_cpu_var(var) per_cpu__##var diff --git a/include/asm-x86_64/percpu.h b/include/asm-x86_64/percpu.h index c6fbb67eac907736dd7d016fdda53df9e3cf1866..5abd48270101e9e11f1540b3084bd69cb2e5c573 100644 --- a/include/asm-x86_64/percpu.h +++ b/include/asm-x86_64/percpu.h @@ -20,6 +20,11 @@ #define DEFINE_PER_CPU(type, name) \ __attribute__((__section__(".data.percpu"))) __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + __attribute__((__section__(".data.percpu.shared_aligned"))) \ + __typeof__(type) per_cpu__##name \ + ____cacheline_internodealigned_in_smp + /* var is in discarded region: offset to particular copy we want */ #define per_cpu(var, cpu) (*({ \ extern int simple_identifier_##var(void); \ @@ -46,6 +51,8 @@ extern void setup_per_cpu_areas(void); #define DEFINE_PER_CPU(type, name) \ __typeof__(type) per_cpu__##name +#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ + DEFINE_PER_CPU(type, name) #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var)) #define __get_cpu_var(var) per_cpu__##var