cpu.c 14.9 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
L
Linus Torvalds 已提交
2
/*
3
 * CPU subsystem support
L
Linus Torvalds 已提交
4 5
 */

6
#include <linux/kernel.h>
L
Linus Torvalds 已提交
7 8
#include <linux/module.h>
#include <linux/init.h>
A
Al Viro 已提交
9
#include <linux/sched.h>
L
Linus Torvalds 已提交
10 11 12
#include <linux/cpu.h>
#include <linux/topology.h>
#include <linux/device.h>
13
#include <linux/node.h>
14
#include <linux/gfp.h>
15
#include <linux/slab.h>
16
#include <linux/percpu.h>
17
#include <linux/acpi.h>
18
#include <linux/of.h>
19
#include <linux/cpufeature.h>
R
Rik van Riel 已提交
20
#include <linux/tick.h>
21
#include <linux/pm_qos.h>
22
#include <linux/sched/isolation.h>
L
Linus Torvalds 已提交
23

24
#include "base.h"
L
Linus Torvalds 已提交
25

26
static DEFINE_PER_CPU(struct device *, cpu_sys_devices);
27

28 29 30 31 32 33 34 35 36
static int cpu_subsys_match(struct device *dev, struct device_driver *drv)
{
	/* ACPI style match is the only one that may succeed. */
	if (acpi_driver_match_device(dev, drv))
		return 1;

	return 0;
}

L
Linus Torvalds 已提交
37
#ifdef CONFIG_HOTPLUG_CPU
38 39 40 41 42 43 44 45 46
static void change_cpu_under_node(struct cpu *cpu,
			unsigned int from_nid, unsigned int to_nid)
{
	int cpuid = cpu->dev.id;
	unregister_cpu_under_node(cpuid, from_nid);
	register_cpu_under_node(cpuid, to_nid);
	cpu->node_id = to_nid;
}

47
static int cpu_subsys_online(struct device *dev)
L
Linus Torvalds 已提交
48
{
49
	struct cpu *cpu = container_of(dev, struct cpu, dev);
50 51
	int cpuid = dev->id;
	int from_nid, to_nid;
52
	int ret;
L
Linus Torvalds 已提交
53

54
	from_nid = cpu_to_node(cpuid);
55
	if (from_nid == NUMA_NO_NODE)
56
		return -ENODEV;
57

Q
Qais Yousef 已提交
58
	ret = cpu_device_up(dev);
59 60 61 62 63 64 65
	/*
	 * When hot adding memory to memoryless node and enabling a cpu
	 * on the node, node number of the cpu may internally change.
	 */
	to_nid = cpu_to_node(cpuid);
	if (from_nid != to_nid)
		change_cpu_under_node(cpu, from_nid, to_nid);
L
Linus Torvalds 已提交
66

67
	return ret;
L
Linus Torvalds 已提交
68 69
}

70
static int cpu_subsys_offline(struct device *dev)
L
Linus Torvalds 已提交
71
{
Q
Qais Yousef 已提交
72
	return cpu_device_down(dev);
L
Linus Torvalds 已提交
73
}
74

75
void unregister_cpu(struct cpu *cpu)
L
Linus Torvalds 已提交
76
{
77
	int logical_cpu = cpu->dev.id;
L
Linus Torvalds 已提交
78

79 80
	unregister_cpu_under_node(logical_cpu, cpu_to_node(logical_cpu));

81
	device_unregister(&cpu->dev);
82
	per_cpu(cpu_sys_devices, logical_cpu) = NULL;
L
Linus Torvalds 已提交
83 84
	return;
}
85 86

#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
87 88
static ssize_t cpu_probe_store(struct device *dev,
			       struct device_attribute *attr,
89
			       const char *buf,
90 91
			       size_t count)
{
92 93 94 95 96 97 98 99 100 101 102
	ssize_t cnt;
	int ret;

	ret = lock_device_hotplug_sysfs();
	if (ret)
		return ret;

	cnt = arch_cpu_probe(buf, count);

	unlock_device_hotplug();
	return cnt;
103 104
}

105 106
static ssize_t cpu_release_store(struct device *dev,
				 struct device_attribute *attr,
107
				 const char *buf,
108 109
				 size_t count)
{
110 111 112 113 114 115 116 117 118 119 120
	ssize_t cnt;
	int ret;

	ret = lock_device_hotplug_sysfs();
	if (ret)
		return ret;

	cnt = arch_cpu_release(buf, count);

	unlock_device_hotplug();
	return cnt;
121 122
}

123 124
static DEVICE_ATTR(probe, S_IWUSR, NULL, cpu_probe_store);
static DEVICE_ATTR(release, S_IWUSR, NULL, cpu_release_store);
125
#endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
L
Linus Torvalds 已提交
126 127
#endif /* CONFIG_HOTPLUG_CPU */

128 129 130
struct bus_type cpu_subsys = {
	.name = "cpu",
	.dev_name = "cpu",
131
	.match = cpu_subsys_match,
132 133 134 135 136 137 138
#ifdef CONFIG_HOTPLUG_CPU
	.online = cpu_subsys_online,
	.offline = cpu_subsys_offline,
#endif
};
EXPORT_SYMBOL_GPL(cpu_subsys);

139 140 141
#ifdef CONFIG_KEXEC
#include <linux/kexec.h>

142 143
static ssize_t crash_notes_show(struct device *dev,
				struct device_attribute *attr,
144
				char *buf)
145
{
146
	struct cpu *cpu = container_of(dev, struct cpu, dev);
147 148 149
	unsigned long long addr;
	int cpunum;

150
	cpunum = cpu->dev.id;
151 152 153 154 155 156 157

	/*
	 * Might be reading other cpu's data based on which cpu read thread
	 * has been scheduled. But cpu data (memory) is allocated once during
	 * boot up and this data does not change there after. Hence this
	 * operation should be safe. No locking required.
	 */
158
	addr = per_cpu_ptr_to_phys(per_cpu_ptr(crash_notes, cpunum));
159 160

	return sysfs_emit(buf, "%llx\n", addr);
161
}
162
static DEVICE_ATTR_ADMIN_RO(crash_notes);
163

164
static ssize_t crash_notes_size_show(struct device *dev,
165 166 167
				     struct device_attribute *attr,
				     char *buf)
{
168
	return sysfs_emit(buf, "%zu\n", sizeof(note_buf_t));
169
}
170
static DEVICE_ATTR_ADMIN_RO(crash_notes_size);
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185

static struct attribute *crash_note_cpu_attrs[] = {
	&dev_attr_crash_notes.attr,
	&dev_attr_crash_notes_size.attr,
	NULL
};

static struct attribute_group crash_note_cpu_attr_group = {
	.attrs = crash_note_cpu_attrs,
};
#endif

static const struct attribute_group *common_cpu_attr_groups[] = {
#ifdef CONFIG_KEXEC
	&crash_note_cpu_attr_group,
186
#endif
187 188
	NULL
};
189

190 191 192 193 194 195 196
static const struct attribute_group *hotplugable_cpu_attr_groups[] = {
#ifdef CONFIG_KEXEC
	&crash_note_cpu_attr_group,
#endif
	NULL
};

197 198 199
/*
 * Print cpu online, possible, present, and system maps
 */
200 201

struct cpu_attr {
202
	struct device_attribute attr;
203
	const struct cpumask *const map;
204 205
};

206 207
static ssize_t show_cpus_attr(struct device *dev,
			      struct device_attribute *attr,
208
			      char *buf)
209
{
210
	struct cpu_attr *ca = container_of(attr, struct cpu_attr, attr);
211

212
	return cpumap_print_to_pagebuf(true, buf, ca->map);
213 214
}

215 216
#define _CPU_ATTR(name, map) \
	{ __ATTR(name, 0444, show_cpus_attr, NULL), map }
217

218
/* Keep in sync with cpu_subsys_attrs */
219
static struct cpu_attr cpu_attrs[] = {
220 221 222
	_CPU_ATTR(online, &__cpu_online_mask),
	_CPU_ATTR(possible, &__cpu_possible_mask),
	_CPU_ATTR(present, &__cpu_present_mask),
223
};
224

225 226 227
/*
 * Print values for NR_CPUS and offlined cpus
 */
228 229
static ssize_t print_cpus_kernel_max(struct device *dev,
				     struct device_attribute *attr, char *buf)
230
{
231
	return sysfs_emit(buf, "%d\n", NR_CPUS - 1);
232
}
233
static DEVICE_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL);
234 235 236 237

/* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
unsigned int total_cpus;

238 239
static ssize_t print_cpus_offline(struct device *dev,
				  struct device_attribute *attr, char *buf)
240
{
241
	int len = 0;
242 243 244 245 246
	cpumask_var_t offline;

	/* display offline cpus < nr_cpu_ids */
	if (!alloc_cpumask_var(&offline, GFP_KERNEL))
		return -ENOMEM;
247
	cpumask_andnot(offline, cpu_possible_mask, cpu_online_mask);
248
	len += sysfs_emit_at(buf, len, "%*pbl", cpumask_pr_args(offline));
249 250 251 252
	free_cpumask_var(offline);

	/* display offline cpus >= nr_cpu_ids */
	if (total_cpus && nr_cpu_ids < total_cpus) {
253
		len += sysfs_emit_at(buf, len, ",");
254 255

		if (nr_cpu_ids == total_cpus-1)
256
			len += sysfs_emit_at(buf, len, "%u", nr_cpu_ids);
257
		else
258 259
			len += sysfs_emit_at(buf, len, "%u-%d",
					     nr_cpu_ids, total_cpus - 1);
260 261
	}

262 263 264
	len += sysfs_emit_at(buf, len, "\n");

	return len;
265
}
266
static DEVICE_ATTR(offline, 0444, print_cpus_offline, NULL);
267

R
Rik van Riel 已提交
268 269 270
static ssize_t print_cpus_isolated(struct device *dev,
				  struct device_attribute *attr, char *buf)
{
271
	int len;
272
	cpumask_var_t isolated;
R
Rik van Riel 已提交
273

274 275 276 277 278
	if (!alloc_cpumask_var(&isolated, GFP_KERNEL))
		return -ENOMEM;

	cpumask_andnot(isolated, cpu_possible_mask,
		       housekeeping_cpumask(HK_FLAG_DOMAIN));
279
	len = sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(isolated));
280 281

	free_cpumask_var(isolated);
R
Rik van Riel 已提交
282

283
	return len;
R
Rik van Riel 已提交
284 285 286
}
static DEVICE_ATTR(isolated, 0444, print_cpus_isolated, NULL);

R
Rik van Riel 已提交
287 288
#ifdef CONFIG_NO_HZ_FULL
static ssize_t print_cpus_nohz_full(struct device *dev,
289
				    struct device_attribute *attr, char *buf)
R
Rik van Riel 已提交
290
{
291
	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(tick_nohz_full_mask));
R
Rik van Riel 已提交
292 293 294 295
}
static DEVICE_ATTR(nohz_full, 0444, print_cpus_nohz_full, NULL);
#endif

296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
static void cpu_device_release(struct device *dev)
{
	/*
	 * This is an empty function to prevent the driver core from spitting a
	 * warning at us.  Yes, I know this is directly opposite of what the
	 * documentation for the driver core and kobjects say, and the author
	 * of this code has already been publically ridiculed for doing
	 * something as foolish as this.  However, at this point in time, it is
	 * the only way to handle the issue of statically allocated cpu
	 * devices.  The different architectures will have their cpu device
	 * code reworked to properly handle this in the near future, so this
	 * function will then be changed to correctly free up the memory held
	 * by the cpu device.
	 *
	 * Never copy this way of doing things, or you too will be made fun of
311
	 * on the linux-kernel list, you have been warned.
312 313 314
	 */
}

315 316 317 318 319
#ifdef CONFIG_GENERIC_CPU_AUTOPROBE
static ssize_t print_cpu_modalias(struct device *dev,
				  struct device_attribute *attr,
				  char *buf)
{
320
	int len = 0;
321 322
	u32 i;

323 324 325
	len += sysfs_emit_at(buf, len,
			     "cpu:type:" CPU_FEATURE_TYPEFMT ":feature:",
			     CPU_FEATURE_TYPEVAL);
326 327 328

	for (i = 0; i < MAX_CPU_FEATURES; i++)
		if (cpu_have_feature(i)) {
329
			if (len + sizeof(",XXXX\n") >= PAGE_SIZE) {
330 331 332
				WARN(1, "CPU features overflow page\n");
				break;
			}
333
			len += sysfs_emit_at(buf, len, ",%04X", i);
334
		}
335 336
	len += sysfs_emit_at(buf, len, "\n");
	return len;
337 338 339 340 341 342 343 344 345 346 347 348 349 350
}

static int cpu_uevent(struct device *dev, struct kobj_uevent_env *env)
{
	char *buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
	if (buf) {
		print_cpu_modalias(NULL, NULL, buf);
		add_uevent_var(env, "MODALIAS=%s", buf);
		kfree(buf);
	}
	return 0;
}
#endif

L
Linus Torvalds 已提交
351
/*
352
 * register_cpu - Setup a sysfs device for a CPU.
353 354
 * @cpu - cpu->hotpluggable field set to 1 will generate a control file in
 *	  sysfs for this CPU.
L
Linus Torvalds 已提交
355 356 357 358
 * @num - CPU number to use when creating the device.
 *
 * Initialize and register the CPU device.
 */
359
int register_cpu(struct cpu *cpu, int num)
L
Linus Torvalds 已提交
360 361
{
	int error;
362

363
	cpu->node_id = cpu_to_node(num);
364
	memset(&cpu->dev, 0x00, sizeof(struct device));
365 366
	cpu->dev.id = num;
	cpu->dev.bus = &cpu_subsys;
367
	cpu->dev.release = cpu_device_release;
368
	cpu->dev.offline_disabled = !cpu->hotpluggable;
369
	cpu->dev.offline = !cpu_online(num);
370
	cpu->dev.of_node = of_get_cpu_node(num, NULL);
371
#ifdef CONFIG_GENERIC_CPU_AUTOPROBE
372
	cpu->dev.bus->uevent = cpu_uevent;
373
#endif
374
	cpu->dev.groups = common_cpu_attr_groups;
375 376
	if (cpu->hotpluggable)
		cpu->dev.groups = hotplugable_cpu_attr_groups;
377
	error = device_register(&cpu->dev);
378 379
	if (error) {
		put_device(&cpu->dev);
A
Alex Shi 已提交
380
		return error;
381
	}
382

A
Alex Shi 已提交
383 384
	per_cpu(cpu_sys_devices, num) = &cpu->dev;
	register_cpu_under_node(num, cpu_to_node(num));
385 386
	dev_pm_qos_expose_latency_limit(&cpu->dev,
					PM_QOS_RESUME_LATENCY_NO_CONSTRAINT);
A
Alex Shi 已提交
387 388

	return 0;
L
Linus Torvalds 已提交
389 390
}

391
struct device *get_cpu_device(unsigned cpu)
392
{
393 394
	if (cpu < nr_cpu_ids && cpu_possible(cpu))
		return per_cpu(cpu_sys_devices, cpu);
395 396 397
	else
		return NULL;
}
398 399
EXPORT_SYMBOL_GPL(get_cpu_device);

400 401 402 403 404
static void device_create_release(struct device *dev)
{
	kfree(dev);
}

405
__printf(4, 0)
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
static struct device *
__cpu_device_create(struct device *parent, void *drvdata,
		    const struct attribute_group **groups,
		    const char *fmt, va_list args)
{
	struct device *dev = NULL;
	int retval = -ENODEV;

	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
	if (!dev) {
		retval = -ENOMEM;
		goto error;
	}

	device_initialize(dev);
	dev->parent = parent;
	dev->groups = groups;
	dev->release = device_create_release;
424
	device_set_pm_not_required(dev);
425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
	dev_set_drvdata(dev, drvdata);

	retval = kobject_set_name_vargs(&dev->kobj, fmt, args);
	if (retval)
		goto error;

	retval = device_add(dev);
	if (retval)
		goto error;

	return dev;

error:
	put_device(dev);
	return ERR_PTR(retval);
}

struct device *cpu_device_create(struct device *parent, void *drvdata,
				 const struct attribute_group **groups,
				 const char *fmt, ...)
{
	va_list vargs;
	struct device *dev;

	va_start(vargs, fmt);
	dev = __cpu_device_create(parent, drvdata, groups, fmt, vargs);
	va_end(vargs);
	return dev;
}
EXPORT_SYMBOL_GPL(cpu_device_create);

456
#ifdef CONFIG_GENERIC_CPU_AUTOPROBE
457
static DEVICE_ATTR(modalias, 0444, print_cpu_modalias, NULL);
458 459
#endif

460 461 462 463 464 465 466 467 468 469
static struct attribute *cpu_root_attrs[] = {
#ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
	&dev_attr_probe.attr,
	&dev_attr_release.attr,
#endif
	&cpu_attrs[0].attr.attr,
	&cpu_attrs[1].attr.attr,
	&cpu_attrs[2].attr.attr,
	&dev_attr_kernel_max.attr,
	&dev_attr_offline.attr,
R
Rik van Riel 已提交
470
	&dev_attr_isolated.attr,
R
Rik van Riel 已提交
471 472 473
#ifdef CONFIG_NO_HZ_FULL
	&dev_attr_nohz_full.attr,
#endif
474
#ifdef CONFIG_GENERIC_CPU_AUTOPROBE
475 476
	&dev_attr_modalias.attr,
#endif
477 478 479 480 481 482 483 484 485 486 487
	NULL
};

static struct attribute_group cpu_root_attr_group = {
	.attrs = cpu_root_attrs,
};

static const struct attribute_group *cpu_root_attr_groups[] = {
	&cpu_root_attr_group,
	NULL,
};
L
Linus Torvalds 已提交
488

489 490
bool cpu_is_hotpluggable(unsigned cpu)
{
491 492
	struct device *dev = get_cpu_device(cpu);
	return dev && container_of(dev, struct cpu, dev)->hotpluggable;
493 494 495
}
EXPORT_SYMBOL_GPL(cpu_is_hotpluggable);

496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
#ifdef CONFIG_GENERIC_CPU_DEVICES
static DEFINE_PER_CPU(struct cpu, cpu_devices);
#endif

static void __init cpu_dev_register_generic(void)
{
#ifdef CONFIG_GENERIC_CPU_DEVICES
	int i;

	for_each_possible_cpu(i) {
		if (register_cpu(&per_cpu(cpu_devices, i), i))
			panic("Failed to register CPU device");
	}
#endif
}

512 513 514 515 516
#ifdef CONFIG_GENERIC_CPU_VULNERABILITIES

ssize_t __weak cpu_show_meltdown(struct device *dev,
				 struct device_attribute *attr, char *buf)
{
517
	return sysfs_emit(buf, "Not affected\n");
518 519 520 521 522
}

ssize_t __weak cpu_show_spectre_v1(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
523
	return sysfs_emit(buf, "Not affected\n");
524 525 526 527 528
}

ssize_t __weak cpu_show_spectre_v2(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
529
	return sysfs_emit(buf, "Not affected\n");
530 531
}

532 533 534
ssize_t __weak cpu_show_spec_store_bypass(struct device *dev,
					  struct device_attribute *attr, char *buf)
{
535
	return sysfs_emit(buf, "Not affected\n");
536 537
}

538 539 540
ssize_t __weak cpu_show_l1tf(struct device *dev,
			     struct device_attribute *attr, char *buf)
{
541
	return sysfs_emit(buf, "Not affected\n");
542 543
}

544 545 546
ssize_t __weak cpu_show_mds(struct device *dev,
			    struct device_attribute *attr, char *buf)
{
547
	return sysfs_emit(buf, "Not affected\n");
548 549
}

550 551 552 553
ssize_t __weak cpu_show_tsx_async_abort(struct device *dev,
					struct device_attribute *attr,
					char *buf)
{
554
	return sysfs_emit(buf, "Not affected\n");
555 556
}

557
ssize_t __weak cpu_show_itlb_multihit(struct device *dev,
558
				      struct device_attribute *attr, char *buf)
559
{
560
	return sysfs_emit(buf, "Not affected\n");
561 562
}

563 564 565
ssize_t __weak cpu_show_srbds(struct device *dev,
			      struct device_attribute *attr, char *buf)
{
566
	return sysfs_emit(buf, "Not affected\n");
567 568
}

569 570 571
static DEVICE_ATTR(meltdown, 0444, cpu_show_meltdown, NULL);
static DEVICE_ATTR(spectre_v1, 0444, cpu_show_spectre_v1, NULL);
static DEVICE_ATTR(spectre_v2, 0444, cpu_show_spectre_v2, NULL);
572
static DEVICE_ATTR(spec_store_bypass, 0444, cpu_show_spec_store_bypass, NULL);
573
static DEVICE_ATTR(l1tf, 0444, cpu_show_l1tf, NULL);
574
static DEVICE_ATTR(mds, 0444, cpu_show_mds, NULL);
575
static DEVICE_ATTR(tsx_async_abort, 0444, cpu_show_tsx_async_abort, NULL);
576
static DEVICE_ATTR(itlb_multihit, 0444, cpu_show_itlb_multihit, NULL);
577
static DEVICE_ATTR(srbds, 0444, cpu_show_srbds, NULL);
578 579 580 581 582

static struct attribute *cpu_root_vulnerabilities_attrs[] = {
	&dev_attr_meltdown.attr,
	&dev_attr_spectre_v1.attr,
	&dev_attr_spectre_v2.attr,
583
	&dev_attr_spec_store_bypass.attr,
584
	&dev_attr_l1tf.attr,
585
	&dev_attr_mds.attr,
586
	&dev_attr_tsx_async_abort.attr,
587
	&dev_attr_itlb_multihit.attr,
588
	&dev_attr_srbds.attr,
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
	NULL
};

static const struct attribute_group cpu_root_vulnerabilities_group = {
	.name  = "vulnerabilities",
	.attrs = cpu_root_vulnerabilities_attrs,
};

static void __init cpu_register_vulnerabilities(void)
{
	if (sysfs_create_group(&cpu_subsys.dev_root->kobj,
			       &cpu_root_vulnerabilities_group))
		pr_err("Unable to register CPU vulnerabilities\n");
}

#else
static inline void cpu_register_vulnerabilities(void) { }
#endif

608
void __init cpu_dev_init(void)
L
Linus Torvalds 已提交
609
{
610 611
	if (subsys_system_register(&cpu_subsys, cpu_root_attr_groups))
		panic("Failed to register CPU subsystem");
612

613
	cpu_dev_register_generic();
614
	cpu_register_vulnerabilities();
L
Linus Torvalds 已提交
615
}