intel_rdt.h 11.8 KB
Newer Older
1 2 3
#ifndef _ASM_X86_INTEL_RDT_H
#define _ASM_X86_INTEL_RDT_H

4
#include <linux/sched.h>
5
#include <linux/kernfs.h>
6 7 8
#include <linux/jump_label.h>

#define IA32_L3_QOS_CFG		0xc81
9
#define IA32_L3_CBM_BASE	0xc90
10
#define IA32_L2_CBM_BASE	0xd10
11
#define IA32_MBA_THRTL_BASE	0xd50
12

13 14
#define L3_QOS_CDP_ENABLE	0x01ULL

15 16 17 18 19 20 21
/*
 * Event IDs are used to program IA32_QM_EVTSEL before reading event
 * counter from IA32_QM_CTR
 */
#define QOS_L3_OCCUP_EVENT_ID		0x01
#define QOS_L3_MBM_TOTAL_EVENT_ID	0x02
#define QOS_L3_MBM_LOCAL_EVENT_ID	0x03
22 23

#define MBM_CNTR_WIDTH			24
24
#define MBM_OVERFLOW_INTERVAL		1000
25

26 27
#define RMID_VAL_ERROR			BIT_ULL(63)
#define RMID_VAL_UNAVAIL		BIT_ULL(62)
28

29 30
DECLARE_STATIC_KEY_FALSE(rdt_enable_key);

31 32 33 34 35 36 37 38 39 40 41
/**
 * struct mon_evt - Entry in the event list of a resource
 * @evtid:		event id
 * @name:		name of the event
 */
struct mon_evt {
	u32			evtid;
	char			*name;
	struct list_head	list;
};

V
Vikas Shivappa 已提交
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
/**
 * struct mon_data_bits - Monitoring details for each event file
 * @rid:               Resource id associated with the event file.
 * @evtid:             Event id associated with the event file
 * @domid:             The domain to which the event file belongs
 */
union mon_data_bits {
	void *priv;
	struct {
		unsigned int rid	: 10;
		unsigned int evtid	: 8;
		unsigned int domid	: 14;
	} u;
};

struct rmid_read {
	struct rdtgroup		*rgrp;
59
	struct rdt_domain	*d;
V
Vikas Shivappa 已提交
60
	int			evtid;
61
	bool			first;
V
Vikas Shivappa 已提交
62 63 64
	u64			val;
};

65 66 67 68
extern unsigned int intel_cqm_threshold;
extern bool rdt_alloc_capable;
extern bool rdt_mon_capable;
extern unsigned int rdt_mon_features;
69 70 71 72 73 74 75 76 77

enum rdt_group_type {
	RDTCTRL_GROUP = 0,
	RDTMON_GROUP,
	RDT_NUM_GROUP,
};

/**
 * struct mongroup - store mon group's data in resctrl fs.
V
Vikas Shivappa 已提交
78
 * @mon_data_kn		kernlfs node for the mon_data directory
79 80 81 82 83
 * @parent:			parent rdtgrp
 * @crdtgrp_list:		child rdtgroup node list
 * @rmid:			rmid for this rdtgroup
 */
struct mongroup {
V
Vikas Shivappa 已提交
84
	struct kernfs_node	*mon_data_kn;
85 86 87 88 89
	struct rdtgroup		*parent;
	struct list_head	crdtgrp_list;
	u32			rmid;
};

90 91 92 93 94
/**
 * struct rdtgroup - store rdtgroup's data in resctrl file system.
 * @kn:				kernfs node
 * @rdtgroup_list:		linked list for all rdtgroups
 * @closid:			closid for this rdtgroup
T
Tony Luck 已提交
95
 * @cpu_mask:			CPUs assigned to this rdtgroup
96 97
 * @flags:			status bits
 * @waitcount:			how many cpus expect to find this
T
Tony Luck 已提交
98
 *				group when they acquire rdtgroup_mutex
99 100 101
 * @type:			indicates type of this rdtgroup - either
 *				monitor only or ctrl_mon group
 * @mon:			mongroup related data
102 103 104 105
 */
struct rdtgroup {
	struct kernfs_node	*kn;
	struct list_head	rdtgroup_list;
106
	u32			closid;
T
Tony Luck 已提交
107
	struct cpumask		cpu_mask;
108 109
	int			flags;
	atomic_t		waitcount;
110 111
	enum rdt_group_type	type;
	struct mongroup		mon;
112 113
};

114 115 116
/* rdtgroup.flags */
#define	RDT_DELETED		1

117 118 119
/* rftype.flags */
#define RFTYPE_FLAGS_CPUS_LIST	1

120 121 122 123 124 125
/*
 * Define the file type flags for base and info directories.
 */
#define RFTYPE_INFO			BIT(0)
#define RFTYPE_BASE			BIT(1)
#define RF_CTRLSHIFT			4
126
#define RF_MONSHIFT			5
127
#define RFTYPE_CTRL			BIT(RF_CTRLSHIFT)
128
#define RFTYPE_MON			BIT(RF_MONSHIFT)
129 130 131
#define RFTYPE_RES_CACHE		BIT(8)
#define RFTYPE_RES_MB			BIT(9)
#define RF_CTRL_INFO			(RFTYPE_INFO | RFTYPE_CTRL)
132
#define RF_MON_INFO			(RFTYPE_INFO | RFTYPE_MON)
133 134
#define RF_CTRL_BASE			(RFTYPE_BASE | RFTYPE_CTRL)

135 136 137
/* List of all resource groups */
extern struct list_head rdt_all_groups;

138 139
extern int max_name_width, max_data_width;

140 141
int __init rdtgroup_init(void);

142 143
/**
 * struct rftype - describe each file in the resctrl file system
144 145 146
 * @name:	File name
 * @mode:	Access mode
 * @kf_ops:	File operations
147
 * @flags:	File specific RFTYPE_FLAGS_* flags
148
 * @fflags:	File specific RF_* or RFTYPE_* flags
149 150
 * @seq_show:	Show content of the file
 * @write:	Write to the file
151 152 153 154 155
 */
struct rftype {
	char			*name;
	umode_t			mode;
	struct kernfs_ops	*kf_ops;
156
	unsigned long		flags;
157
	unsigned long		fflags;
158 159 160 161 162 163 164 165 166 167 168 169

	int (*seq_show)(struct kernfs_open_file *of,
			struct seq_file *sf, void *v);
	/*
	 * write() is the generic write callback which maps directly to
	 * kernfs write operation and overrides all other operations.
	 * Maximum write size is determined by ->max_write_len.
	 */
	ssize_t (*write)(struct kernfs_open_file *of,
			 char *buf, size_t nbytes, loff_t off);
};

170 171 172 173 174 175 176 177 178 179
/**
 * struct mbm_state - status for each MBM counter in each domain
 * @chunks:	Total data moved (multiply by rdt_group.mon_scale to get bytes)
 * @prev_msr	Value of IA32_QM_CTR for this RMID last time we read it
 */
struct mbm_state {
	u64	chunks;
	u64	prev_msr;
};

180 181 182 183 184
/**
 * struct rdt_domain - group of cpus sharing an RDT resource
 * @list:	all instances of this resource
 * @id:		unique id for this instance
 * @cpu_mask:	which cpus share this resource
185 186
 * @rmid_busy_llc:
 *		bitmap of which limbo RMIDs are above threshold
187 188
 * @mbm_total:	saved state for MBM total bandwidth
 * @mbm_local:	saved state for MBM local bandwidth
189 190 191
 * @mbm_over:	worker to periodically read MBM h/w counters
 * @mbm_work_cpu:
 *		worker cpu for MBM h/w counters
192 193 194 195 196 197 198 199
 * @ctrl_val:	array of cache or mem ctrl values (indexed by CLOSID)
 * @new_ctrl:	new ctrl value to be loaded
 * @have_new_ctrl: did user provide new_ctrl for this domain
 */
struct rdt_domain {
	struct list_head	list;
	int			id;
	struct cpumask		cpu_mask;
200
	unsigned long		*rmid_busy_llc;
201 202
	struct mbm_state	*mbm_total;
	struct mbm_state	*mbm_local;
203 204
	struct delayed_work	mbm_over;
	int			mbm_work_cpu;
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
	u32			*ctrl_val;
	u32			new_ctrl;
	bool			have_new_ctrl;
};

/**
 * struct msr_param - set a range of MSRs from a domain
 * @res:       The resource to use
 * @low:       Beginning index from base MSR
 * @high:      End index
 */
struct msr_param {
	struct rdt_resource	*res;
	int			low;
	int			high;
};

222 223 224 225 226 227 228 229
/**
 * struct rdt_cache - Cache allocation related data
 * @cbm_len:		Length of the cache bit mask
 * @min_cbm_bits:	Minimum number of consecutive bits to be set
 * @cbm_idx_mult:	Multiplier of CBM index
 * @cbm_idx_offset:	Offset of CBM index. CBM index is computed by:
 *			closid * cbm_idx_multi + cbm_idx_offset
 *			in a cache bit mask
230 231
 * @shareable_bits:	Bitmask of shareable resource with other
 *			executing entities
232 233 234 235 236 237
 */
struct rdt_cache {
	unsigned int	cbm_len;
	unsigned int	min_cbm_bits;
	unsigned int	cbm_idx_mult;
	unsigned int	cbm_idx_offset;
238
	unsigned int	shareable_bits;
239 240
};

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
/**
 * struct rdt_membw - Memory bandwidth allocation related data
 * @max_delay:		Max throttle delay. Delay is the hardware
 *			representation for memory bandwidth.
 * @min_bw:		Minimum memory bandwidth percentage user can request
 * @bw_gran:		Granularity at which the memory bandwidth is allocated
 * @delay_linear:	True if memory B/W delay is in linear scale
 * @mb_map:		Mapping of memory B/W percentage to memory B/W delay
 */
struct rdt_membw {
	u32		max_delay;
	u32		min_bw;
	u32		bw_gran;
	u32		delay_linear;
	u32		*mb_map;
};

258 259 260 261 262
static inline bool is_llc_occupancy_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_OCCUP_EVENT_ID));
}

263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
static inline bool is_mbm_total_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_MBM_TOTAL_EVENT_ID));
}

static inline bool is_mbm_local_enabled(void)
{
	return (rdt_mon_features & (1 << QOS_L3_MBM_LOCAL_EVENT_ID));
}

static inline bool is_mbm_enabled(void)
{
	return (is_mbm_total_enabled() || is_mbm_local_enabled());
}

278 279 280 281 282 283
static inline bool is_mbm_event(int e)
{
	return (e >= QOS_L3_MBM_TOTAL_EVENT_ID &&
		e <= QOS_L3_MBM_LOCAL_EVENT_ID);
}

284 285
/**
 * struct rdt_resource - attributes of an RDT resource
V
Vikas Shivappa 已提交
286
 * @rid:		The index of the resource
287
 * @alloc_enabled:	Is allocation enabled on this machine
288
 * @mon_enabled:		Is monitoring enabled for this feature
289
 * @alloc_capable:	Is allocation available on this machine
290
 * @mon_capable:		Is monitor feature available on this machine
291 292 293 294 295
 * @name:		Name to use in "schemata" file
 * @num_closid:		Number of CLOSIDs available
 * @cache_level:	Which cache level defines scope of this resource
 * @default_ctrl:	Specifies default cache cbm or memory B/W percent.
 * @msr_base:		Base MSR address for CBMs
296
 * @msr_update:		Function pointer to update QOS MSRs
297 298 299
 * @data_width:		Character width of data when displaying
 * @domains:		All domains for this resource
 * @cache:		Cache allocation related data
300 301
 * @format_str:		Per resource format string to show domain value
 * @parse_ctrlval:	Per resource function pointer to parse control values
302 303 304
 * @evt_list:			List of monitoring events
 * @num_rmid:			Number of RMIDs available
 * @mon_scale:			cqm counter * mon_scale = occupancy in bytes
305
 * @fflags:			flags to choose base and info files
306 307
 */
struct rdt_resource {
V
Vikas Shivappa 已提交
308
	int			rid;
309
	bool			alloc_enabled;
310
	bool			mon_enabled;
311
	bool			alloc_capable;
312
	bool			mon_capable;
313 314
	char			*name;
	int			num_closid;
315
	int			cache_level;
316
	u32			default_ctrl;
317
	unsigned int		msr_base;
318 319
	void (*msr_update)	(struct rdt_domain *d, struct msr_param *m,
				 struct rdt_resource *r);
320
	int			data_width;
321
	struct list_head	domains;
322 323
	struct rdt_cache	cache;
	struct rdt_membw	membw;
324 325 326
	const char		*format_str;
	int (*parse_ctrlval)	(char *buf, struct rdt_resource *r,
				 struct rdt_domain *d);
327 328 329
	struct list_head	evt_list;
	int			num_rmid;
	unsigned int		mon_scale;
330
	unsigned long		fflags;
331 332
};

333
int parse_cbm(char *buf, struct rdt_resource *r, struct rdt_domain *d);
334
int parse_bw(char *buf, struct rdt_resource *r,  struct rdt_domain *d);
335

336 337
extern struct mutex rdtgroup_mutex;

338
extern struct rdt_resource rdt_resources_all[];
339
extern struct rdtgroup rdtgroup_default;
340
DECLARE_STATIC_KEY_FALSE(rdt_alloc_enable_key);
341 342

int __init rdtgroup_init(void);
343 344 345 346 347 348

enum {
	RDT_RESOURCE_L3,
	RDT_RESOURCE_L3DATA,
	RDT_RESOURCE_L3CODE,
	RDT_RESOURCE_L2,
349
	RDT_RESOURCE_MBA,
350 351 352 353 354

	/* Must be the last */
	RDT_NUM_RESOURCES,
};

355 356 357 358 359
#define for_each_capable_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->alloc_capable || r->mon_capable)

360
#define for_each_alloc_capable_rdt_resource(r)				      \
361
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
362
	     r++)							      \
363
		if (r->alloc_capable)
364

365 366 367 368 369
#define for_each_mon_capable_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->mon_capable)

370
#define for_each_alloc_enabled_rdt_resource(r)				      \
371 372
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
373
		if (r->alloc_enabled)
374

375 376 377 378 379
#define for_each_mon_enabled_rdt_resource(r)				      \
	for (r = rdt_resources_all; r < rdt_resources_all + RDT_NUM_RESOURCES;\
	     r++)							      \
		if (r->mon_enabled)

380 381 382 383 384 385 386 387
/* CPUID.(EAX=10H, ECX=ResID=1).EAX */
union cpuid_0x10_1_eax {
	struct {
		unsigned int cbm_len:5;
	} split;
	unsigned int full;
};

388 389 390 391 392 393 394 395
/* CPUID.(EAX=10H, ECX=ResID=3).EAX */
union cpuid_0x10_3_eax {
	struct {
		unsigned int max_delay:12;
	} split;
	unsigned int full;
};

396 397
/* CPUID.(EAX=10H, ECX=ResID).EDX */
union cpuid_0x10_x_edx {
398 399 400 401 402
	struct {
		unsigned int cos_max:16;
	} split;
	unsigned int full;
};
403

404
void rdt_ctrl_update(void *arg);
405 406
struct rdtgroup *rdtgroup_kn_lock_live(struct kernfs_node *kn);
void rdtgroup_kn_unlock(struct kernfs_node *kn);
V
Vikas Shivappa 已提交
407 408
struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
				   struct list_head **pos);
T
Tony Luck 已提交
409 410 411 412
ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
				char *buf, size_t nbytes, loff_t off);
int rdtgroup_schemata_show(struct kernfs_open_file *of,
			   struct seq_file *s, void *v);
413
struct rdt_domain *get_domain_from_cpu(int cpu, struct rdt_resource *r);
414 415
int alloc_rmid(void);
void free_rmid(u32 rmid);
416
int rdt_get_mon_l3_config(struct rdt_resource *r);
V
Vikas Shivappa 已提交
417 418
void mon_event_count(void *info);
int rdtgroup_mondata_show(struct seq_file *m, void *arg);
419 420 421 422
void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
				    unsigned int dom_id);
void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
				    struct rdt_domain *d);
423 424
void mon_event_read(struct rmid_read *rr, struct rdt_domain *d,
		    struct rdtgroup *rdtgrp, int evtid, int first);
425
void mbm_setup_overflow_handler(struct rdt_domain *dom, unsigned long delay_ms);
426
void mbm_handle_overflow(struct work_struct *work);
F
Fenghua Yu 已提交
427

428
#endif /* _ASM_X86_INTEL_RDT_H */