intel_rdt.h 13.4 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
2 3 4
#ifndef _ASM_X86_INTEL_RDT_H
#define _ASM_X86_INTEL_RDT_H

5
#include <linux/sched.h>
6
#include <linux/kernfs.h>
7 8 9
#include <linux/jump_label.h>

#define IA32_L3_QOS_CFG		0xc81
10
#define IA32_L2_QOS_CFG		0xc82
11
#define IA32_L3_CBM_BASE	0xc90
12
#define IA32_L2_CBM_BASE	0xd10
13
#define IA32_MBA_THRTL_BASE	0xd50
14

15 16
#define L3_QOS_CDP_ENABLE	0x01ULL

17 18
#define L2_QOS_CDP_ENABLE	0x01ULL

19 20 21 22 23 24 25
/*
 * Event IDs are used to program IA32_QM_EVTSEL before reading event
 * counter from IA32_QM_CTR
 */
#define QOS_L3_OCCUP_EVENT_ID		0x01
#define QOS_L3_MBM_TOTAL_EVENT_ID	0x02
#define QOS_L3_MBM_LOCAL_EVENT_ID	0x03
26

27 28
#define CQM_LIMBOCHECK_INTERVAL	1000

29
#define MBM_CNTR_WIDTH			24
30
#define MBM_OVERFLOW_INTERVAL		1000
31
#define MAX_MBA_BW			100u
32

33 34
#define RMID_VAL_ERROR			BIT_ULL(63)
#define RMID_VAL_UNAVAIL		BIT_ULL(62)
35

36 37
DECLARE_STATIC_KEY_FALSE(rdt_enable_key);

38 39 40 41 42 43 44 45 46 47 48
/**
 * struct mon_evt - Entry in the event list of a resource
 * @evtid:		event id
 * @name:		name of the event
 */
struct mon_evt {
	u32			evtid;
	char			*name;
	struct list_head	list;
};

V
Vikas Shivappa 已提交
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
/**
 * struct mon_data_bits - Monitoring details for each event file
 * @rid:               Resource id associated with the event file.
 * @evtid:             Event id associated with the event file
 * @domid:             The domain to which the event file belongs
 */
union mon_data_bits {
	void *priv;
	struct {
		unsigned int rid	: 10;
		unsigned int evtid	: 8;
		unsigned int domid	: 14;
	} u;
};

struct rmid_read {
	struct rdtgroup		*rgrp;
66
	struct rdt_domain	*d;
V
Vikas Shivappa 已提交
67
	int			evtid;
68
	bool			first;
V
Vikas Shivappa 已提交
69 70 71
	u64			val;
};

72 73 74 75
extern unsigned int intel_cqm_threshold;
extern bool rdt_alloc_capable;
extern bool rdt_mon_capable;
extern unsigned int rdt_mon_features;
76 77 78 79 80 81 82 83 84

enum rdt_group_type {
	RDTCTRL_GROUP = 0,
	RDTMON_GROUP,
	RDT_NUM_GROUP,
};

/**
 * struct mongroup - store mon group's data in resctrl fs.
V
Vikas Shivappa 已提交
85
 * @mon_data_kn		kernlfs node for the mon_data directory
86 87 88 89 90
 * @parent:			parent rdtgrp
 * @crdtgrp_list:		child rdtgroup node list
 * @rmid:			rmid for this rdtgroup
 */
struct mongroup {
V
Vikas Shivappa 已提交
91
	struct kernfs_node	*mon_data_kn;
92 93 94 95 96
	struct rdtgroup		*parent;
	struct list_head	crdtgrp_list;
	u32			rmid;
};

97 98 99 100 101
/**
 * struct rdtgroup - store rdtgroup's data in resctrl file system.
 * @kn:				kernfs node
 * @rdtgroup_list:		linked list for all rdtgroups
 * @closid:			closid for this rdtgroup
T
Tony Luck 已提交
102
 * @cpu_mask:			CPUs assigned to this rdtgroup
103 104
 * @flags:			status bits
 * @waitcount:			how many cpus expect to find this
T
Tony Luck 已提交
105
 *				group when they acquire rdtgroup_mutex
106 107 108
 * @type:			indicates type of this rdtgroup - either
 *				monitor only or ctrl_mon group
 * @mon:			mongroup related data
109 110 111 112
 */
struct rdtgroup {
	struct kernfs_node	*kn;
	struct list_head	rdtgroup_list;
113
	u32			closid;
T
Tony Luck 已提交
114
	struct cpumask		cpu_mask;
115 116
	int			flags;
	atomic_t		waitcount;
117 118
	enum rdt_group_type	type;
	struct mongroup		mon;
119 120
};

121 122 123
/* rdtgroup.flags */
#define	RDT_DELETED		1

124 125 126
/* rftype.flags */
#define RFTYPE_FLAGS_CPUS_LIST	1

127 128 129 130 131 132
/*
 * Define the file type flags for base and info directories.
 */
#define RFTYPE_INFO			BIT(0)
#define RFTYPE_BASE			BIT(1)
#define RF_CTRLSHIFT			4
133
#define RF_MONSHIFT			5
134
#define RF_TOPSHIFT			6
135
#define RFTYPE_CTRL			BIT(RF_CTRLSHIFT)
136
#define RFTYPE_MON			BIT(RF_MONSHIFT)
137
#define RFTYPE_TOP			BIT(RF_TOPSHIFT)
138 139 140
#define RFTYPE_RES_CACHE		BIT(8)
#define RFTYPE_RES_MB			BIT(9)
#define RF_CTRL_INFO			(RFTYPE_INFO | RFTYPE_CTRL)
141
#define RF_MON_INFO			(RFTYPE_INFO | RFTYPE_MON)
142
#define RF_TOP_INFO			(RFTYPE_INFO | RFTYPE_TOP)
143 144
#define RF_CTRL_BASE			(RFTYPE_BASE | RFTYPE_CTRL)

145 146 147
/* List of all resource groups */
extern struct list_head rdt_all_groups;

148 149
extern int max_name_width, max_data_width;

150 151
int __init rdtgroup_init(void);

152 153
/**
 * struct rftype - describe each file in the resctrl file system
154 155 156
 * @name:	File name
 * @mode:	Access mode
 * @kf_ops:	File operations
157
 * @flags:	File specific RFTYPE_FLAGS_* flags
158
 * @fflags:	File specific RF_* or RFTYPE_* flags
159 160
 * @seq_show:	Show content of the file
 * @write:	Write to the file
161 162 163 164 165
 */
struct rftype {
	char			*name;
	umode_t			mode;
	struct kernfs_ops	*kf_ops;
166
	unsigned long		flags;
167
	unsigned long		fflags;
168 169 170 171 172 173 174 175 176 177 178 179

	int (*seq_show)(struct kernfs_open_file *of,
			struct seq_file *sf, void *v);
	/*
	 * write() is the generic write callback which maps directly to
	 * kernfs write operation and overrides all other operations.
	 * Maximum write size is determined by ->max_write_len.
	 */
	ssize_t (*write)(struct kernfs_open_file *of,
			 char *buf, size_t nbytes, loff_t off);
};

180 181 182 183
/**
 * struct mbm_state - status for each MBM counter in each domain
 * @chunks:	Total data moved (multiply by rdt_group.mon_scale to get bytes)
 * @prev_msr	Value of IA32_QM_CTR for this RMID last time we read it
184 185 186 187 188
 * @chunks_bw	Total local data moved. Used for bandwidth calculation
 * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
 * @prev_bw	The most recent bandwidth in MBps
 * @delta_bw	Difference between the current and previous bandwidth
 * @delta_comp	Indicates whether to compute the delta_bw
189 190 191 192
 */
struct mbm_state {
	u64	chunks;
	u64	prev_msr;
193 194 195 196 197
	u64	chunks_bw;
	u64	prev_bw_msr;
	u32	prev_bw;
	u32	delta_bw;
	bool	delta_comp;
198 199
};

200 201 202 203 204
/**
 * struct rdt_domain - group of cpus sharing an RDT resource
 * @list:	all instances of this resource
 * @id:		unique id for this instance
 * @cpu_mask:	which cpus share this resource
205 206
 * @rmid_busy_llc:
 *		bitmap of which limbo RMIDs are above threshold
207 208
 * @mbm_total:	saved state for MBM total bandwidth
 * @mbm_local:	saved state for MBM local bandwidth
209
 * @mbm_over:	worker to periodically read MBM h/w counters
210
 * @cqm_limbo:	worker to periodically read CQM h/w counters
211 212
 * @mbm_work_cpu:
 *		worker cpu for MBM h/w counters
213 214
 * @cqm_work_cpu:
 *		worker cpu for CQM h/w counters
215
 * @ctrl_val:	array of cache or mem ctrl values (indexed by CLOSID)
216
 * @mbps_val:	When mba_sc is enabled, this holds the bandwidth in MBps
217 218 219 220 221 222 223
 * @new_ctrl:	new ctrl value to be loaded
 * @have_new_ctrl: did user provide new_ctrl for this domain
 */
struct rdt_domain {
	struct list_head	list;
	int			id;
	struct cpumask		cpu_mask;
224
	unsigned long		*rmid_busy_llc;
225 226
	struct mbm_state	*mbm_total;
	struct mbm_state	*mbm_local;
227
	struct delayed_work	mbm_over;
228
	struct delayed_work	cqm_limbo;
229
	int			mbm_work_cpu;
230
	int			cqm_work_cpu;
231
	u32			*ctrl_val;
232
	u32			*mbps_val;
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
	u32			new_ctrl;
	bool			have_new_ctrl;
};

/**
 * struct msr_param - set a range of MSRs from a domain
 * @res:       The resource to use
 * @low:       Beginning index from base MSR
 * @high:      End index
 */
struct msr_param {
	struct rdt_resource	*res;
	int			low;
	int			high;
};

249 250 251 252 253 254 255 256
/**
 * struct rdt_cache - Cache allocation related data
 * @cbm_len:		Length of the cache bit mask
 * @min_cbm_bits:	Minimum number of consecutive bits to be set
 * @cbm_idx_mult:	Multiplier of CBM index
 * @cbm_idx_offset:	Offset of CBM index. CBM index is computed by:
 *			closid * cbm_idx_multi + cbm_idx_offset
 *			in a cache bit mask
257 258
 * @shareable_bits:	Bitmask of shareable resource with other
 *			executing entities
259 260 261 262 263 264
 */
struct rdt_cache {
	unsigned int	cbm_len;
	unsigned int	min_cbm_bits;
	unsigned int	cbm_idx_mult;
	unsigned int	cbm_idx_offset;
265
	unsigned int	shareable_bits;
266 267
};

268 269 270 271 272 273 274
/**
 * struct rdt_membw - Memory bandwidth allocation related data
 * @max_delay:		Max throttle delay. Delay is the hardware
 *			representation for memory bandwidth.
 * @min_bw:		Minimum memory bandwidth percentage user can request
 * @bw_gran:		Granularity at which the memory bandwidth is allocated
 * @delay_linear:	True if memory B/W delay is in linear scale
275
 * @mba_sc:		True if MBA software controller(mba_sc) is enabled
276 277 278 279 280 281 282
 * @mb_map:		Mapping of memory B/W percentage to memory B/W delay
 */
struct rdt_membw {
	u32		max_delay;
	u32		min_bw;
	u32		bw_gran;
	u32		delay_linear;
283
	bool		mba_sc;
284 285 286
	u32		*mb_map;
};

287 288 289 290 291
static inline bool is_llc_occupancy_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
}

292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
static inline bool is_mbm_total_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
}

static inline bool is_mbm_local_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
}

static inline bool is_mbm_enabled(void)
{
	return (is_mbm_total_enabled() || is_mbm_local_enabled());
}

307 308 309 310 311 312
static inline bool is_mbm_event(int e)
{
	return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
		e <= QOS_L3_MBM_LOCAL_EVENT_ID);
}

313 314
/**
 * struct rdt_resource - attributes of an RDT resource
V
Vikas Shivappa 已提交
315
 * @rid:		The index of the resource
316
 * @alloc_enabled:	Is allocation enabled on this machine
317
 * @mon_enabled:		Is monitoring enabled for this feature
318
 * @alloc_capable:	Is allocation available on this machine
319
 * @mon_capable:		Is monitor feature available on this machine
320 321 322 323 324
 * @name:		Name to use in "schemata" file
 * @num_closid:		Number of CLOSIDs available
 * @cache_level:	Which cache level defines scope of this resource
 * @default_ctrl:	Specifies default cache cbm or memory B/W percent.
 * @msr_base:		Base MSR address for CBMs
325
 * @msr_update:		Function pointer to update QOS MSRs
326 327 328
 * @data_width:		Character width of data when displaying
 * @domains:		All domains for this resource
 * @cache:		Cache allocation related data
329 330
 * @format_str:		Per resource format string to show domain value
 * @parse_ctrlval:	Per resource function pointer to parse control values
331 332 333
 * @evt_list:			List of monitoring events
 * @num_rmid:			Number of RMIDs available
 * @mon_scale:			cqm counter * mon_scale = occupancy in bytes
334
 * @fflags:			flags to choose base and info files
335 336
 */
struct rdt_resource {
V
Vikas Shivappa 已提交
337
	int			rid;
338
	bool			alloc_enabled;
339
	bool			mon_enabled;
340
	bool			alloc_capable;
341
	bool			mon_capable;
342 343
	char			*name;
	int			num_closid;
344
	int			cache_level;
345
	u32			default_ctrl;
346
	unsigned int		msr_base;
347 348
	void (*msr_update)	(struct rdt_domain *d, struct msr_param *m,
				 struct rdt_resource *r);
349
	int			data_width;
350
	struct list_head	domains;
351 352
	struct rdt_cache	cache;
	struct rdt_membw	membw;
353 354 355
	const char		*format_str;
	int (*parse_ctrlval)	(char *buf, struct rdt_resource *r,
				 struct rdt_domain *d);
356 357 358
	struct list_head	evt_list;
	int			num_rmid;
	unsigned int		mon_scale;
359
	unsigned long		fflags;
360 361
};

362
int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d);
363
int parse_bw(char *buf, struct rdt_resource *r,  struct rdt_domain *d);
364

365 366
extern struct mutex rdtgroup_mutex;

367
extern struct rdt_resource rdt_resources_all[];
368
extern struct rdtgroup rdtgroup_default;
369
DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
370 371

int __init rdtgroup_init(void);
372 373 374 375 376 377

enum {
	RDT_RESOURCE_L3,
	RDT_RESOURCE_L3DATA,
	RDT_RESOURCE_L3CODE,
	RDT_RESOURCE_L2,
378 379
	RDT_RESOURCE_L2DATA,
	RDT_RESOURCE_L2CODE,
380
	RDT_RESOURCE_MBA,
381 382 383 384 385

	/* Must be the last */
	RDT_NUM_RESOURCES,
};

386 387 388 389 390
#define for_each_capable_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->alloc_capable || r->mon_capable)

391
#define for_each_alloc_capable_rdt_resource(r)				      \
392
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
393
	     r++)							      \
394
		if (r->alloc_capable)
395

396 397 398 399 400
#define for_each_mon_capable_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->mon_capable)

401
#define for_each_alloc_enabled_rdt_resource(r)				      \
402 403
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
404
		if (r->alloc_enabled)
405

406 407 408 409 410
#define for_each_mon_enabled_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->mon_enabled)

411 412 413 414 415 416 417 418
/* CPUID.(EAX=10H, ECX=ResID=1).EAX */
union cpuid_0x10_1_eax {
	struct {
		unsigned int cbm_len:5;
	} split;
	unsigned int full;
};

419 420 421 422 423 424 425 426
/* CPUID.(EAX=10H, ECX=ResID=3).EAX */
union cpuid_0x10_3_eax {
	struct {
		unsigned int max_delay:12;
	} split;
	unsigned int full;
};

427 428
/* CPUID.(EAX=10H, ECX=ResID).EDX */
union cpuid_0x10_x_edx {
429 430 431 432 433
	struct {
		unsigned int cos_max:16;
	} split;
	unsigned int full;
};
434

435 436 437 438
void rdt_last_cmd_clear(void);
void rdt_last_cmd_puts(const char *s);
void rdt_last_cmd_printf(const char *fmt, ...);

439
void rdt_ctrl_update(void *arg);
440 441
struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
void rdtgroup_kn_unlock(struct kernfs_node *kn);
V
Vikas Shivappa 已提交
442 443
struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
				   struct list_head **pos);
T
Tony Luck 已提交
444 445 446 447
ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off);
int rdtgroup_schemata_show(struct kernfs_open_file *of,
			   struct seq_file *s, void *v);
448
struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
449 450
int alloc_rmid(void);
void free_rmid(u32 rmid);
451
int rdt_get_mon_l3_config(struct rdt_resource *r);
V
Vikas Shivappa 已提交
452 453
void mon_event_count(void *info);
int rdtgroup_mondata_show(struct seq_file *m, void *arg);
454 455 456 457
void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
				    unsigned int dom_id);
void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
				    struct rdt_domain *d);
458 459
void mon_event_read(struct rmid_read *rr, struct rdt_domain *d,
		    struct rdtgroup *rdtgrp, int evtid, int first);
460 461
void mbm_setup_overflow_handler(struct rdt_domain *dom,
				unsigned long delay_ms);
462
void mbm_handle_overflow(struct work_struct *work);
463
bool is_mba_sc(struct rdt_resource *r);
464
void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
465
u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
466 467 468 469
void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
void cqm_handle_limbo(struct work_struct *work);
bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
void __check_limbo(struct rdt_domain *d, bool force_free);
F
Fenghua Yu 已提交
470

471
#endif /* _ASM_X86_INTEL_RDT_H */