cpufeature.h 25.6 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0-only */
2 3 4 5 6 7 8
/*
 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#ifndef __ASM_CPUFEATURE_H
#define __ASM_CPUFEATURE_H

9
#include <asm/cpucaps.h>
10
#include <asm/cputype.h>
11
#include <asm/hwcap.h>
12
#include <asm/sysreg.h>
13

14 15
#define MAX_CPU_FEATURES	64
#define cpu_feature(x)		KERNEL_HWCAP_ ## x
16

17
#ifndef __ASSEMBLY__
18

19 20
#include <linux/bug.h>
#include <linux/jump_label.h>
21 22
#include <linux/kernel.h>

23 24 25 26 27 28 29 30 31 32 33 34 35 36
/*
 * CPU feature register tracking
 *
 * The safe value of a CPUID feature field is dependent on the implications
 * of the values assigned to it by the architecture. Based on the relationship
 * between the values, the features are classified into 3 types - LOWER_SAFE,
 * HIGHER_SAFE and EXACT.
 *
 * The lowest value of all the CPUs is chosen for LOWER_SAFE and highest
 * for HIGHER_SAFE. It is expected that all CPUs have the same value for
 * a field when EXACT is specified, failing which, the safe value specified
 * in the table is chosen.
 */

37
enum ftr_type {
38 39 40 41
	FTR_EXACT,			/* Use a predefined safe value */
	FTR_LOWER_SAFE,			/* Smaller value is safe */
	FTR_HIGHER_SAFE,		/* Bigger value is safe */
	FTR_HIGHER_OR_ZERO_SAFE,	/* Bigger value is safe, but 0 is biggest */
42 43 44 45 46
};

#define FTR_STRICT	true	/* SANITY check strict matching required */
#define FTR_NONSTRICT	false	/* SANITY check ignored */

47 48 49
#define FTR_SIGNED	true	/* Value should be treated as signed */
#define FTR_UNSIGNED	false	/* Value should be treated as unsigned */

50 51 52
#define FTR_VISIBLE	true	/* Feature visible to the user space */
#define FTR_HIDDEN	false	/* Feature is hidden from the user */

53 54 55
#define FTR_VISIBLE_IF_IS_ENABLED(config)		\
	(IS_ENABLED(config) ? FTR_VISIBLE : FTR_HIDDEN)

56
struct arm64_ftr_bits {
57
	bool		sign;	/* Value is signed ? */
58
	bool		visible;
59
	bool		strict;	/* CPU Sanity check: strict matching required ? */
60 61 62
	enum ftr_type	type;
	u8		shift;
	u8		width;
63
	s64		safe_val; /* safe value for FTR_EXACT features */
64 65 66 67 68 69 70 71
};

/*
 * @arm64_ftr_reg - Feature register
 * @strict_mask		Bits which should match across all CPUs for sanity.
 * @sys_val		Safe value across the CPUs (system view)
 */
struct arm64_ftr_reg {
72 73
	const char			*name;
	u64				strict_mask;
74
	u64				user_mask;
75
	u64				sys_val;
76
	u64				user_val;
77
	const struct arm64_ftr_bits	*ftr_bits;
78 79
};

80 81
extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;

82 83
int arm64_cpu_ftr_regs_traverse(int (*op)(u32, u64, void *), void *argp);

84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
/*
 * CPU capabilities:
 *
 * We use arm64_cpu_capabilities to represent system features, errata work
 * arounds (both used internally by kernel and tracked in cpu_hwcaps) and
 * ELF HWCAPs (which are exposed to user).
 *
 * To support systems with heterogeneous CPUs, we need to make sure that we
 * detect the capabilities correctly on the system and take appropriate
 * measures to ensure there are no incompatibilities.
 *
 * This comment tries to explain how we treat the capabilities.
 * Each capability has the following list of attributes :
 *
 * 1) Scope of Detection : The system detects a given capability by
 *    performing some checks at runtime. This could be, e.g, checking the
 *    value of a field in CPU ID feature register or checking the cpu
 *    model. The capability provides a call back ( @matches() ) to
 *    perform the check. Scope defines how the checks should be performed.
103
 *    There are three cases:
104 105 106 107 108 109 110 111 112 113 114 115
 *
 *     a) SCOPE_LOCAL_CPU: check all the CPUs and "detect" if at least one
 *        matches. This implies, we have to run the check on all the
 *        booting CPUs, until the system decides that state of the
 *        capability is finalised. (See section 2 below)
 *		Or
 *     b) SCOPE_SYSTEM: check all the CPUs and "detect" if all the CPUs
 *        matches. This implies, we run the check only once, when the
 *        system decides to finalise the state of the capability. If the
 *        capability relies on a field in one of the CPU ID feature
 *        registers, we use the sanitised value of the register from the
 *        CPU feature infrastructure to make the decision.
116 117 118 119 120
 *		Or
 *     c) SCOPE_BOOT_CPU: Check only on the primary boot CPU to detect the
 *        feature. This category is for features that are "finalised"
 *        (or used) by the kernel very early even before the SMP cpus
 *        are brought up.
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
 *
 *    The process of detection is usually denoted by "update" capability
 *    state in the code.
 *
 * 2) Finalise the state : The kernel should finalise the state of a
 *    capability at some point during its execution and take necessary
 *    actions if any. Usually, this is done, after all the boot-time
 *    enabled CPUs are brought up by the kernel, so that it can make
 *    better decision based on the available set of CPUs. However, there
 *    are some special cases, where the action is taken during the early
 *    boot by the primary boot CPU. (e.g, running the kernel at EL2 with
 *    Virtualisation Host Extensions). The kernel usually disallows any
 *    changes to the state of a capability once it finalises the capability
 *    and takes any action, as it may be impossible to execute the actions
 *    safely. A CPU brought up after a capability is "finalised" is
 *    referred to as "Late CPU" w.r.t the capability. e.g, all secondary
 *    CPUs are treated "late CPUs" for capabilities determined by the boot
 *    CPU.
 *
140 141 142 143 144
 *    At the moment there are two passes of finalising the capabilities.
 *      a) Boot CPU scope capabilities - Finalised by primary boot CPU via
 *         setup_boot_cpu_capabilities().
 *      b) Everything except (a) - Run via setup_system_capabilities().
 *
145 146 147 148 149 150 151 152
 * 3) Verification: When a CPU is brought online (e.g, by user or by the
 *    kernel), the kernel should make sure that it is safe to use the CPU,
 *    by verifying that the CPU is compliant with the state of the
 *    capabilities finalised already. This happens via :
 *
 *	secondary_start_kernel()-> check_local_cpu_capabilities()
 *
 *    As explained in (2) above, capabilities could be finalised at
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
 *    different points in the execution. Each newly booted CPU is verified
 *    against the capabilities that have been finalised by the time it
 *    boots.
 *
 *	a) SCOPE_BOOT_CPU : All CPUs are verified against the capability
 *	except for the primary boot CPU.
 *
 *	b) SCOPE_LOCAL_CPU, SCOPE_SYSTEM: All CPUs hotplugged on by the
 *	user after the kernel boot are verified against the capability.
 *
 *    If there is a conflict, the kernel takes an action, based on the
 *    severity (e.g, a CPU could be prevented from booting or cause a
 *    kernel panic). The CPU is allowed to "affect" the state of the
 *    capability, if it has not been finalised already. See section 5
 *    for more details on conflicts.
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
 *
 * 4) Action: As mentioned in (2), the kernel can take an action for each
 *    detected capability, on all CPUs on the system. Appropriate actions
 *    include, turning on an architectural feature, modifying the control
 *    registers (e.g, SCTLR, TCR etc.) or patching the kernel via
 *    alternatives. The kernel patching is batched and performed at later
 *    point. The actions are always initiated only after the capability
 *    is finalised. This is usally denoted by "enabling" the capability.
 *    The actions are initiated as follows :
 *	a) Action is triggered on all online CPUs, after the capability is
 *	finalised, invoked within the stop_machine() context from
 *	enable_cpu_capabilitie().
 *
 *	b) Any late CPU, brought up after (1), the action is triggered via:
 *
 *	  check_local_cpu_capabilities() -> verify_local_cpu_capabilities()
 *
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
 * 5) Conflicts: Based on the state of the capability on a late CPU vs.
 *    the system state, we could have the following combinations :
 *
 *		x-----------------------------x
 *		| Type  | System   | Late CPU |
 *		|-----------------------------|
 *		|  a    |   y      |    n     |
 *		|-----------------------------|
 *		|  b    |   n      |    y     |
 *		x-----------------------------x
 *
 *     Two separate flag bits are defined to indicate whether each kind of
 *     conflict can be allowed:
 *		ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU - Case(a) is allowed
 *		ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU - Case(b) is allowed
 *
 *     Case (a) is not permitted for a capability that the system requires
 *     all CPUs to have in order for the capability to be enabled. This is
 *     typical for capabilities that represent enhanced functionality.
 *
 *     Case (b) is not permitted for a capability that must be enabled
 *     during boot if any CPU in the system requires it in order to run
 *     safely. This is typical for erratum work arounds that cannot be
 *     enabled after the corresponding capability is finalised.
 *
 *     In some non-typical cases either both (a) and (b), or neither,
 *     should be permitted. This can be described by including neither
 *     or both flags in the capability's type field.
213 214 215 216
 *
 *     In case of a conflict, the CPU is prevented from booting. If the
 *     ARM64_CPUCAP_PANIC_ON_CONFLICT flag is specified for the capability,
 *     then a kernel panic is triggered.
217 218 219
 */


220 221 222 223
/*
 * Decide how the capability is detected.
 * On any local CPU vs System wide vs the primary boot CPU
 */
224 225
#define ARM64_CPUCAP_SCOPE_LOCAL_CPU		((u16)BIT(0))
#define ARM64_CPUCAP_SCOPE_SYSTEM		((u16)BIT(1))
226 227 228 229 230 231
/*
 * The capabilitiy is detected on the Boot CPU and is used by kernel
 * during early boot. i.e, the capability should be "detected" and
 * "enabled" as early as possibly on all booting CPUs.
 */
#define ARM64_CPUCAP_SCOPE_BOOT_CPU		((u16)BIT(2))
232 233
#define ARM64_CPUCAP_SCOPE_MASK			\
	(ARM64_CPUCAP_SCOPE_SYSTEM	|	\
234 235
	 ARM64_CPUCAP_SCOPE_LOCAL_CPU	|	\
	 ARM64_CPUCAP_SCOPE_BOOT_CPU)
236 237 238

#define SCOPE_SYSTEM				ARM64_CPUCAP_SCOPE_SYSTEM
#define SCOPE_LOCAL_CPU				ARM64_CPUCAP_SCOPE_LOCAL_CPU
239
#define SCOPE_BOOT_CPU				ARM64_CPUCAP_SCOPE_BOOT_CPU
240
#define SCOPE_ALL				ARM64_CPUCAP_SCOPE_MASK
241

242 243 244 245 246 247 248
/*
 * Is it permitted for a late CPU to have this capability when system
 * hasn't already enabled it ?
 */
#define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU	((u16)BIT(4))
/* Is it safe for a late CPU to miss this capability when system has it */
#define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU	((u16)BIT(5))
249 250
/* Panic when a conflict is detected */
#define ARM64_CPUCAP_PANIC_ON_CONFLICT		((u16)BIT(6))
251 252 253 254 255 256 257 258 259 260 261 262 263

/*
 * CPU errata workarounds that need to be enabled at boot time if one or
 * more CPUs in the system requires it. When one of these capabilities
 * has been enabled, it is safe to allow any CPU to boot that doesn't
 * require the workaround. However, it is not safe if a "late" CPU
 * requires a workaround and the system hasn't enabled it already.
 */
#define ARM64_CPUCAP_LOCAL_CPU_ERRATUM		\
	(ARM64_CPUCAP_SCOPE_LOCAL_CPU | ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU)
/*
 * CPU feature detected at boot time based on system-wide value of a
 * feature. It is safe for a late CPU to have this feature even though
W
Will Deacon 已提交
264
 * the system hasn't enabled it, although the feature will not be used
265 266 267 268 269
 * by Linux in this case. If the system has enabled this feature already,
 * then every late CPU must have it.
 */
#define ARM64_CPUCAP_SYSTEM_FEATURE	\
	(ARM64_CPUCAP_SCOPE_SYSTEM | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
270 271 272
/*
 * CPU feature detected at boot time based on feature of one or more CPUs.
 * All possible conflicts for a late CPU are ignored.
273 274
 * NOTE: this means that a late CPU with the feature will *not* cause the
 * capability to be advertised by cpus_have_*cap()!
275 276 277 278 279
 */
#define ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE		\
	(ARM64_CPUCAP_SCOPE_LOCAL_CPU		|	\
	 ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU	|	\
	 ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
280

281 282 283 284 285 286 287 288 289
/*
 * CPU feature detected at boot time, on one or more CPUs. A late CPU
 * is not allowed to have the capability when the system doesn't have it.
 * It is Ok for a late CPU to miss the feature.
 */
#define ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE	\
	(ARM64_CPUCAP_SCOPE_LOCAL_CPU		|	\
	 ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU)

290 291
/*
 * CPU feature used early in the boot based on the boot CPU. All secondary
292 293
 * CPUs must match the state of the capability as detected by the boot CPU. In
 * case of a conflict, a kernel panic is triggered.
294
 */
295 296
#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE		\
	(ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PANIC_ON_CONFLICT)
297

298 299 300 301 302
/*
 * CPU feature used early in the boot based on the boot CPU. It is safe for a
 * late CPU to have this feature even though the boot CPU hasn't enabled it,
 * although the feature will not be used by Linux in this case. If the boot CPU
 * has enabled this feature already, then every late CPU must have it.
303
 */
304 305
#define ARM64_CPUCAP_BOOT_CPU_FEATURE                  \
	(ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
306

307 308 309
struct arm64_cpu_capabilities {
	const char *desc;
	u16 capability;
310
	u16 type;
311
	bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
312
	/*
313 314 315 316 317 318 319 320 321 322
	 * Take the appropriate actions to configure this capability
	 * for this CPU. If the capability is detected by the kernel
	 * this will be called on all the CPUs in the system,
	 * including the hotplugged CPUs, regardless of whether the
	 * capability is available on that specific CPU. This is
	 * useful for some capabilities (e.g, working around CPU
	 * errata), where all the CPUs must take some action (e.g,
	 * changing system control/configuration). Thus, if an action
	 * is required only if the CPU has the capability, then the
	 * routine must check it before taking any action.
323 324
	 */
	void (*cpu_enable)(const struct arm64_cpu_capabilities *cap);
325 326
	union {
		struct {	/* To be used for erratum handling only */
327
			struct midr_range midr_range;
328 329 330 331
			const struct arm64_midr_revidr {
				u32 midr_rv;		/* revision/variant */
				u32 revidr_mask;
			} * const fixed_revs;
332
		};
333

334
		const struct midr_range *midr_range_list;
335
		struct {	/* Feature register checking */
336
			u32 sys_reg;
337 338 339 340
			u8 field_pos;
			u8 min_field_value;
			u8 hwcap_type;
			bool sign;
341
			unsigned long hwcap;
342
		};
343
	};
344 345 346 347 348 349 350 351 352 353 354 355 356

	/*
	 * An optional list of "matches/cpu_enable" pair for the same
	 * "capability" of the same "type" as described by the parent.
	 * Only matches(), cpu_enable() and fields relevant to these
	 * methods are significant in the list. The cpu_enable is
	 * invoked only if the corresponding entry "matches()".
	 * However, if a cpu_enable() method is associated
	 * with multiple matches(), care should be taken that either
	 * the match criteria are mutually exclusive, or that the
	 * method is robust against being called multiple times.
	 */
	const struct arm64_cpu_capabilities *match_list;
357 358
};

359 360 361 362 363
static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap)
{
	return cap->type & ARM64_CPUCAP_SCOPE_MASK;
}

364
/*
365
 * Generic helper for handling capabilities with multiple (match,enable) pairs
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
 * of call backs, sharing the same capability bit.
 * Iterate over each entry to see if at least one matches.
 */
static inline bool
cpucap_multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry,
			       int scope)
{
	const struct arm64_cpu_capabilities *caps;

	for (caps = entry->match_list; caps->matches; caps++)
		if (caps->matches(caps, scope))
			return true;

	return false;
}

382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
static __always_inline bool is_vhe_hyp_code(void)
{
	/* Only defined for code run in VHE hyp context */
	return __is_defined(__KVM_VHE_HYPERVISOR__);
}

static __always_inline bool is_nvhe_hyp_code(void)
{
	/* Only defined for code run in NVHE hyp context */
	return __is_defined(__KVM_NVHE_HYPERVISOR__);
}

static __always_inline bool is_hyp_code(void)
{
	return is_vhe_hyp_code() || is_nvhe_hyp_code();
}

399
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
400
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
401
extern struct static_key_false arm64_const_caps_ready;
402

403 404 405 406
/* ARM64 CAPS + alternative_cb */
#define ARM64_NPATCHABLE (ARM64_NCAPS + 1)
extern DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);

407 408 409
#define for_each_available_cap(cap)		\
	for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)

410
bool this_cpu_has_cap(unsigned int cap);
411 412 413 414
void cpu_set_feature(unsigned int num);
bool cpu_have_feature(unsigned int num);
unsigned long cpu_get_elf_hwcap(void);
unsigned long cpu_get_elf_hwcap2(void);
415

416 417
#define cpu_set_named_feature(name) cpu_set_feature(cpu_feature(name))
#define cpu_have_named_feature(name) cpu_have_feature(cpu_feature(name))
418

419
static __always_inline bool system_capabilities_finalized(void)
420
{
421
	return static_branch_likely(&arm64_const_caps_ready);
422 423
}

424 425 426 427 428
/*
 * Test for a capability with a runtime check.
 *
 * Before the capability is detected, this returns false.
 */
429 430
static inline bool cpus_have_cap(unsigned int num)
{
431
	if (num >= ARM64_NCAPS)
432
		return false;
433
	return test_bit(num, cpu_hwcaps);
434 435
}

436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
/*
 * Test for a capability without a runtime check.
 *
 * Before capabilities are finalized, this returns false.
 * After capabilities are finalized, this is patched to avoid a runtime check.
 *
 * @num must be a compile-time constant.
 */
static __always_inline bool __cpus_have_const_cap(int num)
{
	if (num >= ARM64_NCAPS)
		return false;
	return static_branch_unlikely(&cpu_hwcap_keys[num]);
}

/*
452
 * Test for a capability without a runtime check.
453
 *
454
 * Before capabilities are finalized, this will BUG().
455 456 457 458
 * After capabilities are finalized, this is patched to avoid a runtime check.
 *
 * @num must be a compile-time constant.
 */
459
static __always_inline bool cpus_have_final_cap(int num)
460
{
461
	if (system_capabilities_finalized())
462 463
		return __cpus_have_const_cap(num);
	else
464
		BUG();
465 466
}

467
/*
468
 * Test for a capability, possibly with a runtime check for non-hyp code.
469
 *
470 471 472
 * For hyp code, this behaves the same as cpus_have_final_cap().
 *
 * For non-hyp code:
473
 * Before capabilities are finalized, this behaves as cpus_have_cap().
474 475 476 477
 * After capabilities are finalized, this is patched to avoid a runtime check.
 *
 * @num must be a compile-time constant.
 */
478
static __always_inline bool cpus_have_const_cap(int num)
479
{
480 481 482
	if (is_hyp_code())
		return cpus_have_final_cap(num);
	else if (system_capabilities_finalized())
483 484
		return __cpus_have_const_cap(num);
	else
485
		return cpus_have_cap(num);
486 487
}

488 489
static inline void cpus_set_cap(unsigned int num)
{
490
	if (num >= ARM64_NCAPS) {
491
		pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n",
492
			num, ARM64_NCAPS);
493
	} else {
494
		__set_bit(num, cpu_hwcaps);
495
	}
496 497
}

498
static inline int __attribute_const__
499
cpuid_feature_extract_signed_field_width(u64 features, int field, int width)
500
{
501 502 503 504
	return (s64)(features << (64 - width - field)) >> (64 - width);
}

static inline int __attribute_const__
505
cpuid_feature_extract_signed_field(u64 features, int field)
506
{
507
	return cpuid_feature_extract_signed_field_width(features, field, 4);
508 509
}

510
static __always_inline unsigned int __attribute_const__
511 512 513 514 515
cpuid_feature_extract_unsigned_field_width(u64 features, int field, int width)
{
	return (u64)(features << (64 - width - field)) >> (64 - width);
}

516
static __always_inline unsigned int __attribute_const__
517 518 519 520 521
cpuid_feature_extract_unsigned_field(u64 features, int field)
{
	return cpuid_feature_extract_unsigned_field_width(features, field, 4);
}

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544
/*
 * Fields that identify the version of the Performance Monitors Extension do
 * not follow the standard ID scheme. See ARM DDI 0487E.a page D13-2825,
 * "Alternative ID scheme used for the Performance Monitors Extension version".
 */
static inline u64 __attribute_const__
cpuid_feature_cap_perfmon_field(u64 features, int field, u64 cap)
{
	u64 val = cpuid_feature_extract_unsigned_field(features, field);
	u64 mask = GENMASK_ULL(field + 3, field);

	/* Treat IMPLEMENTATION DEFINED functionality as unimplemented */
	if (val == 0xf)
		val = 0;

	if (val > cap) {
		features &= ~mask;
		features |= (cap << field) & mask;
	}

	return features;
}

545
static inline u64 arm64_ftr_mask(const struct arm64_ftr_bits *ftrp)
546 547 548 549
{
	return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift);
}

550 551 552 553 554
static inline u64 arm64_ftr_reg_user_value(const struct arm64_ftr_reg *reg)
{
	return (reg->user_val | (reg->sys_val & reg->user_mask));
}

555
static inline int __attribute_const__
556
cpuid_feature_extract_field_width(u64 features, int field, int width, bool sign)
557 558
{
	return (sign) ?
559 560 561 562 563 564 565 566
		cpuid_feature_extract_signed_field_width(features, field, width) :
		cpuid_feature_extract_unsigned_field_width(features, field, width);
}

static inline int __attribute_const__
cpuid_feature_extract_field(u64 features, int field, bool sign)
{
	return cpuid_feature_extract_field_width(features, field, 4, sign);
567 568
}

569
static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val)
570
{
571
	return (s64)cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width, ftrp->sign);
572 573
}

574
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
575
{
576 577
	return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 ||
		cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1;
578 579
}

580 581 582 583 584 585 586
static inline bool id_aa64pfr0_32bit_el1(u64 pfr0)
{
	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_SHIFT);

	return val == ID_AA64PFR0_EL1_32BIT_64BIT;
}

587 588 589 590 591 592 593
static inline bool id_aa64pfr0_32bit_el0(u64 pfr0)
{
	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT);

	return val == ID_AA64PFR0_EL0_32BIT_64BIT;
}

594 595 596 597 598 599 600
static inline bool id_aa64pfr0_sve(u64 pfr0)
{
	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);

	return val > 0;
}

601
void __init setup_cpu_features(void);
602 603
void check_local_cpu_capabilities(void);

604
u64 read_sanitised_ftr_reg(u32 id);
605

606 607 608 609 610
static inline bool cpu_supports_mixed_endian_el0(void)
{
	return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
}

611 612
static inline bool system_supports_32bit_el0(void)
{
613
	return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
614 615
}

616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
static inline bool system_supports_4kb_granule(void)
{
	u64 mmfr0;
	u32 val;

	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
	val = cpuid_feature_extract_unsigned_field(mmfr0,
						ID_AA64MMFR0_TGRAN4_SHIFT);

	return val == ID_AA64MMFR0_TGRAN4_SUPPORTED;
}

static inline bool system_supports_64kb_granule(void)
{
	u64 mmfr0;
	u32 val;

	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
	val = cpuid_feature_extract_unsigned_field(mmfr0,
						ID_AA64MMFR0_TGRAN64_SHIFT);

	return val == ID_AA64MMFR0_TGRAN64_SUPPORTED;
}

static inline bool system_supports_16kb_granule(void)
{
	u64 mmfr0;
	u32 val;

	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
	val = cpuid_feature_extract_unsigned_field(mmfr0,
						ID_AA64MMFR0_TGRAN16_SHIFT);

	return val == ID_AA64MMFR0_TGRAN16_SUPPORTED;
}

652 653
static inline bool system_supports_mixed_endian_el0(void)
{
654
	return id_aa64mmfr0_mixed_endian_el0(read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1));
655
}
656

657 658 659 660 661 662 663 664 665 666 667 668
static inline bool system_supports_mixed_endian(void)
{
	u64 mmfr0;
	u32 val;

	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
	val = cpuid_feature_extract_unsigned_field(mmfr0,
						ID_AA64MMFR0_BIGENDEL_SHIFT);

	return val == 0x1;
}

669
static __always_inline bool system_supports_fpsimd(void)
670 671 672 673
{
	return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD);
}

674 675 676
static inline bool system_uses_ttbr0_pan(void)
{
	return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) &&
677
		!cpus_have_const_cap(ARM64_HAS_PAN);
678 679
}

680
static __always_inline bool system_supports_sve(void)
681
{
682 683
	return IS_ENABLED(CONFIG_ARM64_SVE) &&
		cpus_have_const_cap(ARM64_SVE);
684 685
}

686
static __always_inline bool system_supports_cnp(void)
687 688 689 690 691
{
	return IS_ENABLED(CONFIG_ARM64_CNP) &&
		cpus_have_const_cap(ARM64_HAS_CNP);
}

692 693 694
static inline bool system_supports_address_auth(void)
{
	return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
695
		cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH);
696 697 698 699 700
}

static inline bool system_supports_generic_auth(void)
{
	return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
701
		cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH);
702 703
}

704
static __always_inline bool system_uses_irq_prio_masking(void)
705 706 707 708 709
{
	return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) &&
	       cpus_have_const_cap(ARM64_HAS_IRQ_PRIO_MASKING);
}

710 711 712 713 714 715
static inline bool system_supports_mte(void)
{
	return IS_ENABLED(CONFIG_ARM64_MTE) &&
		cpus_have_const_cap(ARM64_MTE);
}

716 717 718 719 720 721
static inline bool system_has_prio_mask_debugging(void)
{
	return IS_ENABLED(CONFIG_ARM64_DEBUG_PRIORITY_MASKING) &&
	       system_uses_irq_prio_masking();
}

722 723
static inline bool system_supports_bti(void)
{
724
	return IS_ENABLED(CONFIG_ARM64_BTI) && cpus_have_const_cap(ARM64_BTI);
725 726
}

727 728 729 730 731 732
static inline bool system_supports_tlb_range(void)
{
	return IS_ENABLED(CONFIG_ARM64_TLB_RANGE) &&
		cpus_have_const_cap(ARM64_HAS_TLB_RANGE);
}

733
extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
734

735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754
static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
{
	switch (parange) {
	case 0: return 32;
	case 1: return 36;
	case 2: return 40;
	case 3: return 42;
	case 4: return 44;
	case 5: return 48;
	case 6: return 52;
	/*
	 * A future PE could use a value unknown to the kernel.
	 * However, by the "D10.1.4 Principles of the ID scheme
	 * for fields in ID registers", ARM DDI 0487C.a, any new
	 * value is guaranteed to be higher than what we know already.
	 * As a safe limit, we return the limit supported by the kernel.
	 */
	default: return CONFIG_ARM64_PA_BITS;
	}
}
755 756 757 758 759 760 761 762 763 764 765 766 767 768

/* Check whether hardware update of the Access flag is supported */
static inline bool cpu_has_hw_af(void)
{
	u64 mmfr1;

	if (!IS_ENABLED(CONFIG_ARM64_HW_AFDBM))
		return false;

	mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
	return cpuid_feature_extract_unsigned_field(mmfr1,
						ID_AA64MMFR1_HADBS_SHIFT);
}

769 770 771
#ifdef CONFIG_ARM64_AMU_EXTN
/* Check whether the cpu supports the Activity Monitors Unit (AMU) */
extern bool cpu_has_amu_feat(int cpu);
772 773 774 775 776
#else
static inline bool cpu_has_amu_feat(int cpu)
{
	return false;
}
777 778
#endif

779 780 781
/* Get a cpu that supports the Activity Monitors Unit (AMU) */
extern int get_cpu_with_amu_feat(void);

782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
static inline unsigned int get_vmid_bits(u64 mmfr1)
{
	int vmid_bits;

	vmid_bits = cpuid_feature_extract_unsigned_field(mmfr1,
						ID_AA64MMFR1_VMIDBITS_SHIFT);
	if (vmid_bits == ID_AA64MMFR1_VMIDBITS_16)
		return 16;

	/*
	 * Return the default here even if any reserved
	 * value is fetched from the system register.
	 */
	return 8;
}

u32 get_kvm_ipa_limit(void);
799
void dump_cpu_features(void);
800

801 802
#endif /* __ASSEMBLY__ */

803
#endif