perf_event_p4.c 28.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Netburst Perfomance Events (P4, old Xeon)
 *
 *  Copyright (C) 2010 Parallels, Inc., Cyrill Gorcunov <gorcunov@openvz.org>
 *  Copyright (C) 2010 Intel Corporation, Lin Ming <ming.m.lin@intel.com>
 *
 *  For licencing details see kernel-base/COPYING
 */

#ifdef CONFIG_CPU_SUP_INTEL

#include <asm/perf_event_p4.h>

14
#define P4_CNTR_LIMIT 3
15 16 17
/*
 * array indices: 0,1 - HT threads, used with HT enabled cpu
 */
18 19 20
struct p4_event_bind {
	unsigned int opcode;			/* Event code and ESCR selector */
	unsigned int escr_msr[2];		/* ESCR MSR for this event */
21
	char cntr[2][P4_CNTR_LIMIT];		/* counter index (offset), -1 on abscence */
22 23
};

24
struct p4_pebs_bind {
25 26
	unsigned int metric_pebs;
	unsigned int metric_vert;
27 28
};

29 30 31 32 33
/* it sets P4_PEBS_ENABLE_UOP_TAG as well */
#define P4_GEN_PEBS_BIND(name, pebs, vert)			\
	[P4_PEBS_METRIC__##name] = {				\
		.metric_pebs = pebs | P4_PEBS_ENABLE_UOP_TAG,	\
		.metric_vert = vert,				\
34 35
	}

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
/*
 * note we have P4_PEBS_ENABLE_UOP_TAG always set here
 *
 * it's needed for mapping P4_PEBS_CONFIG_METRIC_MASK bits of
 * event configuration to find out which values are to be
 * written into MSR_IA32_PEBS_ENABLE and MSR_P4_PEBS_MATRIX_VERT
 * resgisters
 */
static struct p4_pebs_bind p4_pebs_bind_map[] = {
	P4_GEN_PEBS_BIND(1stl_cache_load_miss_retired,	0x0000001, 0x0000001),
	P4_GEN_PEBS_BIND(2ndl_cache_load_miss_retired,	0x0000002, 0x0000001),
	P4_GEN_PEBS_BIND(dtlb_load_miss_retired,	0x0000004, 0x0000001),
	P4_GEN_PEBS_BIND(dtlb_store_miss_retired,	0x0000004, 0x0000002),
	P4_GEN_PEBS_BIND(dtlb_all_miss_retired,		0x0000004, 0x0000003),
	P4_GEN_PEBS_BIND(tagged_mispred_branch,		0x0018000, 0x0000010),
	P4_GEN_PEBS_BIND(mob_load_replay_retired,	0x0000200, 0x0000001),
	P4_GEN_PEBS_BIND(split_load_retired,		0x0000400, 0x0000001),
	P4_GEN_PEBS_BIND(split_store_retired,		0x0000400, 0x0000002),
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
};

/*
 * Note that we don't use CCCR1 here, there is an
 * exception for P4_BSQ_ALLOCATION but we just have
 * no workaround
 *
 * consider this binding as resources which particular
 * event may borrow, it doesn't contain EventMask,
 * Tags and friends -- they are left to a caller
 */
static struct p4_event_bind p4_event_bind_map[] = {
	[P4_EVENT_TC_DELIVER_MODE] = {
		.opcode		= P4_OPCODE(P4_EVENT_TC_DELIVER_MODE),
		.escr_msr	= { MSR_P4_TC_ESCR0, MSR_P4_TC_ESCR1 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_BPU_FETCH_REQUEST] = {
		.opcode		= P4_OPCODE(P4_EVENT_BPU_FETCH_REQUEST),
		.escr_msr	= { MSR_P4_BPU_ESCR0, MSR_P4_BPU_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_ITLB_REFERENCE] = {
		.opcode		= P4_OPCODE(P4_EVENT_ITLB_REFERENCE),
		.escr_msr	= { MSR_P4_ITLB_ESCR0, MSR_P4_ITLB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_MEMORY_CANCEL] = {
		.opcode		= P4_OPCODE(P4_EVENT_MEMORY_CANCEL),
		.escr_msr	= { MSR_P4_DAC_ESCR0, MSR_P4_DAC_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_MEMORY_COMPLETE] = {
		.opcode		= P4_OPCODE(P4_EVENT_MEMORY_COMPLETE),
		.escr_msr	= { MSR_P4_SAAT_ESCR0 , MSR_P4_SAAT_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_LOAD_PORT_REPLAY] = {
		.opcode		= P4_OPCODE(P4_EVENT_LOAD_PORT_REPLAY),
		.escr_msr	= { MSR_P4_SAAT_ESCR0, MSR_P4_SAAT_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_STORE_PORT_REPLAY] = {
		.opcode		= P4_OPCODE(P4_EVENT_STORE_PORT_REPLAY),
		.escr_msr	= { MSR_P4_SAAT_ESCR0 ,  MSR_P4_SAAT_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_MOB_LOAD_REPLAY] = {
		.opcode		= P4_OPCODE(P4_EVENT_MOB_LOAD_REPLAY),
		.escr_msr	= { MSR_P4_MOB_ESCR0, MSR_P4_MOB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_PAGE_WALK_TYPE] = {
		.opcode		= P4_OPCODE(P4_EVENT_PAGE_WALK_TYPE),
		.escr_msr	= { MSR_P4_PMH_ESCR0, MSR_P4_PMH_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_BSQ_CACHE_REFERENCE] = {
		.opcode		= P4_OPCODE(P4_EVENT_BSQ_CACHE_REFERENCE),
		.escr_msr	= { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_IOQ_ALLOCATION] = {
		.opcode		= P4_OPCODE(P4_EVENT_IOQ_ALLOCATION),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_IOQ_ACTIVE_ENTRIES] = {	/* shared ESCR */
		.opcode		= P4_OPCODE(P4_EVENT_IOQ_ACTIVE_ENTRIES),
		.escr_msr	= { MSR_P4_FSB_ESCR1,  MSR_P4_FSB_ESCR1 },
		.cntr		= { {2, -1, -1}, {3, -1, -1} },
	},
	[P4_EVENT_FSB_DATA_ACTIVITY] = {
		.opcode		= P4_OPCODE(P4_EVENT_FSB_DATA_ACTIVITY),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_BSQ_ALLOCATION] = {		/* shared ESCR, broken CCCR1 */
		.opcode		= P4_OPCODE(P4_EVENT_BSQ_ALLOCATION),
		.escr_msr	= { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR0 },
		.cntr		= { {0, -1, -1}, {1, -1, -1} },
	},
	[P4_EVENT_BSQ_ACTIVE_ENTRIES] = {	/* shared ESCR */
		.opcode		= P4_OPCODE(P4_EVENT_BSQ_ACTIVE_ENTRIES),
		.escr_msr	= { MSR_P4_BSU_ESCR1 , MSR_P4_BSU_ESCR1 },
		.cntr		= { {2, -1, -1}, {3, -1, -1} },
	},
	[P4_EVENT_SSE_INPUT_ASSIST] = {
		.opcode		= P4_OPCODE(P4_EVENT_SSE_INPUT_ASSIST),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_PACKED_SP_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_PACKED_SP_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_PACKED_DP_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_PACKED_DP_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_SCALAR_SP_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_SCALAR_SP_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_SCALAR_DP_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_SCALAR_DP_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_64BIT_MMX_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_64BIT_MMX_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_128BIT_MMX_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_128BIT_MMX_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_X87_FP_UOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_X87_FP_UOP),
		.escr_msr	= { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_TC_MISC] = {
		.opcode		= P4_OPCODE(P4_EVENT_TC_MISC),
		.escr_msr	= { MSR_P4_TC_ESCR0, MSR_P4_TC_ESCR1 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_GLOBAL_POWER_EVENTS] = {
		.opcode		= P4_OPCODE(P4_EVENT_GLOBAL_POWER_EVENTS),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_TC_MS_XFER] = {
		.opcode		= P4_OPCODE(P4_EVENT_TC_MS_XFER),
		.escr_msr	= { MSR_P4_MS_ESCR0, MSR_P4_MS_ESCR1 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_UOP_QUEUE_WRITES] = {
		.opcode		= P4_OPCODE(P4_EVENT_UOP_QUEUE_WRITES),
		.escr_msr	= { MSR_P4_MS_ESCR0, MSR_P4_MS_ESCR1 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE] = {
		.opcode		= P4_OPCODE(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE),
		.escr_msr	= { MSR_P4_TBPU_ESCR0 , MSR_P4_TBPU_ESCR0 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_RETIRED_BRANCH_TYPE] = {
		.opcode		= P4_OPCODE(P4_EVENT_RETIRED_BRANCH_TYPE),
		.escr_msr	= { MSR_P4_TBPU_ESCR0 , MSR_P4_TBPU_ESCR1 },
		.cntr		= { {4, 5, -1}, {6, 7, -1} },
	},
	[P4_EVENT_RESOURCE_STALL] = {
		.opcode		= P4_OPCODE(P4_EVENT_RESOURCE_STALL),
		.escr_msr	= { MSR_P4_ALF_ESCR0, MSR_P4_ALF_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_WC_BUFFER] = {
		.opcode		= P4_OPCODE(P4_EVENT_WC_BUFFER),
		.escr_msr	= { MSR_P4_DAC_ESCR0, MSR_P4_DAC_ESCR1 },
		.cntr		= { {8, 9, -1}, {10, 11, -1} },
	},
	[P4_EVENT_B2B_CYCLES] = {
		.opcode		= P4_OPCODE(P4_EVENT_B2B_CYCLES),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_BNR] = {
		.opcode		= P4_OPCODE(P4_EVENT_BNR),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_SNOOP] = {
		.opcode		= P4_OPCODE(P4_EVENT_SNOOP),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_RESPONSE] = {
		.opcode		= P4_OPCODE(P4_EVENT_RESPONSE),
		.escr_msr	= { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
		.cntr		= { {0, -1, -1}, {2, -1, -1} },
	},
	[P4_EVENT_FRONT_END_EVENT] = {
		.opcode		= P4_OPCODE(P4_EVENT_FRONT_END_EVENT),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_EXECUTION_EVENT] = {
		.opcode		= P4_OPCODE(P4_EVENT_EXECUTION_EVENT),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_REPLAY_EVENT] = {
		.opcode		= P4_OPCODE(P4_EVENT_REPLAY_EVENT),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_INSTR_RETIRED] = {
		.opcode		= P4_OPCODE(P4_EVENT_INSTR_RETIRED),
		.escr_msr	= { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_UOPS_RETIRED] = {
		.opcode		= P4_OPCODE(P4_EVENT_UOPS_RETIRED),
		.escr_msr	= { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_UOP_TYPE] = {
		.opcode		= P4_OPCODE(P4_EVENT_UOP_TYPE),
		.escr_msr	= { MSR_P4_RAT_ESCR0, MSR_P4_RAT_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_BRANCH_RETIRED] = {
		.opcode		= P4_OPCODE(P4_EVENT_BRANCH_RETIRED),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_MISPRED_BRANCH_RETIRED] = {
		.opcode		= P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED),
		.escr_msr	= { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_X87_ASSIST] = {
		.opcode		= P4_OPCODE(P4_EVENT_X87_ASSIST),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_MACHINE_CLEAR] = {
		.opcode		= P4_OPCODE(P4_EVENT_MACHINE_CLEAR),
		.escr_msr	= { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
	[P4_EVENT_INSTR_COMPLETED] = {
		.opcode		= P4_OPCODE(P4_EVENT_INSTR_COMPLETED),
		.escr_msr	= { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
		.cntr		= { {12, 13, 16}, {14, 15, 17} },
	},
};
297

298
#define P4_GEN_CACHE_EVENT(event, bit, metric)				  \
299 300
	p4_config_pack_escr(P4_ESCR_EVENT(event)			| \
			    P4_ESCR_EMASK_BIT(event, bit))		| \
301
	p4_config_pack_cccr(metric					| \
302
			    P4_CCCR_ESEL(P4_OPCODE_ESEL(P4_OPCODE(event))))
303

304
static __initconst const u64 p4_hw_cache_event_ids
305 306 307 308 309 310 311
				[PERF_COUNT_HW_CACHE_MAX]
				[PERF_COUNT_HW_CACHE_OP_MAX]
				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
{
 [ C(L1D ) ] = {
	[ C(OP_READ) ] = {
		[ C(RESULT_ACCESS) ] = 0x0,
312
		[ C(RESULT_MISS)   ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
313
						P4_PEBS_METRIC__1stl_cache_load_miss_retired),
314 315 316 317 318
	},
 },
 [ C(LL  ) ] = {
	[ C(OP_READ) ] = {
		[ C(RESULT_ACCESS) ] = 0x0,
319
		[ C(RESULT_MISS)   ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
320
						P4_PEBS_METRIC__2ndl_cache_load_miss_retired),
321
	},
322
},
323 324 325
 [ C(DTLB) ] = {
	[ C(OP_READ) ] = {
		[ C(RESULT_ACCESS) ] = 0x0,
326
		[ C(RESULT_MISS)   ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
327
						P4_PEBS_METRIC__dtlb_load_miss_retired),
328 329 330
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = 0x0,
331
		[ C(RESULT_MISS)   ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
332
						P4_PEBS_METRIC__dtlb_store_miss_retired),
333 334 335 336
	},
 },
 [ C(ITLB) ] = {
	[ C(OP_READ) ] = {
337
		[ C(RESULT_ACCESS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_ITLB_REFERENCE, HIT,
338
						P4_PEBS_METRIC__none),
339
		[ C(RESULT_MISS)   ] = P4_GEN_CACHE_EVENT(P4_EVENT_ITLB_REFERENCE, MISS,
340
						P4_PEBS_METRIC__none),
341 342 343 344 345 346 347 348 349 350 351 352
	},
	[ C(OP_WRITE) ] = {
		[ C(RESULT_ACCESS) ] = -1,
		[ C(RESULT_MISS)   ] = -1,
	},
	[ C(OP_PREFETCH) ] = {
		[ C(RESULT_ACCESS) ] = -1,
		[ C(RESULT_MISS)   ] = -1,
	},
 },
};

353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403
static u64 p4_general_events[PERF_COUNT_HW_MAX] = {
  /* non-halted CPU clocks */
  [PERF_COUNT_HW_CPU_CYCLES] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)),

  /*
   * retired instructions
   * in a sake of simplicity we don't use the FSB tagging
   */
  [PERF_COUNT_HW_INSTRUCTIONS] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_INSTR_RETIRED)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, NBOGUSNTAG)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, BOGUSNTAG)),

  /* cache hits */
  [PERF_COUNT_HW_CACHE_REFERENCES] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_BSQ_CACHE_REFERENCE)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITS)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITE)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITM)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITS)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITE)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITM)),

  /* cache misses */
  [PERF_COUNT_HW_CACHE_MISSES] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_BSQ_CACHE_REFERENCE)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_MISS)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_MISS)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, WR_2ndL_MISS)),

  /* branch instructions retired */
  [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_RETIRED_BRANCH_TYPE)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CONDITIONAL)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CALL)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, RETURN)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, INDIRECT)),

  /* mispredicted branches retired */
  [PERF_COUNT_HW_BRANCH_MISSES]	=
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_MISPRED_BRANCH_RETIRED)	|
		P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS)),

  /* bus ready clocks (cpu is driving #DRDY_DRV\#DRDY_OWN):  */
  [PERF_COUNT_HW_BUS_CYCLES] =
	p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_FSB_DATA_ACTIVITY)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_DRV)		|
		P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OWN))	|
	p4_config_pack_cccr(P4_CCCR_EDGE | P4_CCCR_COMPARE),
404 405
};

406 407 408 409 410 411 412 413 414 415 416
static struct p4_event_bind *p4_config_get_bind(u64 config)
{
	unsigned int evnt = p4_config_unpack_event(config);
	struct p4_event_bind *bind = NULL;

	if (evnt < ARRAY_SIZE(p4_event_bind_map))
		bind = &p4_event_bind_map[evnt];

	return bind;
}

417 418
static u64 p4_pmu_event_map(int hw_event)
{
419 420
	struct p4_event_bind *bind;
	unsigned int esel;
421 422
	u64 config;

423 424 425 426
	config = p4_general_events[hw_event];
	bind = p4_config_get_bind(config);
	esel = P4_OPCODE_ESEL(bind->opcode);
	config |= p4_config_pack_cccr(P4_CCCR_ESEL(esel));
427 428 429 430

	return config;
}

431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
static int p4_validate_raw_event(struct perf_event *event)
{
	unsigned int v;

	/* user data may have out-of-bound event index */
	v = p4_config_unpack_event(event->attr.config);
	if (v >= ARRAY_SIZE(p4_event_bind_map)) {
		pr_warning("P4 PMU: Unknown event code: %d\n", v);
		return -EINVAL;
	}

	/*
	 * it may have some screwed PEBS bits
	 */
	if (p4_config_pebs_has(event->attr.config, P4_PEBS_CONFIG_ENABLE)) {
		pr_warning("P4 PMU: PEBS are not supported yet\n");
		return -EINVAL;
	}
	v = p4_config_unpack_metric(event->attr.config);
	if (v >= ARRAY_SIZE(p4_pebs_bind_map)) {
		pr_warning("P4 PMU: Unknown metric code: %d\n", v);
		return -EINVAL;
	}

	return 0;
}

458
static int p4_hw_config(struct perf_event *event)
459
{
460 461
	int cpu = get_cpu();
	int rc = 0;
462
	u32 escr, cccr;
463 464 465 466 467 468 469

	/*
	 * the reason we use cpu that early is that: if we get scheduled
	 * first time on the same cpu -- we will not need swap thread
	 * specific flags in config (and will save some cpu cycles)
	 */

470
	cccr = p4_default_cccr_conf(cpu);
471 472 473 474
	escr = p4_default_escr_conf(cpu, event->attr.exclude_kernel,
					 event->attr.exclude_user);
	event->hw.config = p4_config_pack_escr(escr) |
			   p4_config_pack_cccr(cccr);
475

476
	if (p4_ht_active() && p4_ht_thread(cpu))
477 478
		event->hw.config = p4_set_ht_bit(event->hw.config);

479
	if (event->attr.type == PERF_TYPE_RAW) {
480

481 482
		rc = p4_validate_raw_event(event);
		if (rc)
483 484
			goto out;

485 486 487 488 489 490
		/*
		 * We don't control raw events so it's up to the caller
		 * to pass sane values (and we don't count the thread number
		 * on HT machine but allow HT-compatible specifics to be
		 * passed on)
		 *
491 492 493
		 * Note that for RAW events we allow user to use P4_CCCR_RESERVED
		 * bits since we keep additional info here (for cache events and etc)
		 *
494 495 496 497 498
		 * XXX: HT wide things should check perf_paranoid_cpu() &&
		 *      CAP_SYS_ADMIN
		 */
		event->hw.config |= event->attr.config &
			(p4_config_pack_escr(P4_ESCR_MASK_HT) |
499
			 p4_config_pack_cccr(P4_CCCR_MASK_HT | P4_CCCR_RESERVED));
500
	}
501

502
	rc = x86_setup_perfctr(event);
503
out:
504 505
	put_cpu();
	return rc;
506 507
}

508
static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
509
{
510 511
	int overflow = 0;
	u32 low, high;
512

513 514 515
	rdmsr(hwc->config_base + hwc->idx, low, high);

	/* we need to check high bit for unflagged overflows */
516
	if ((low & P4_CCCR_OVF) || !(high & (1 << 31))) {
517
		overflow = 1;
518
		(void)checking_wrmsrl(hwc->config_base + hwc->idx,
519
			((u64)low) & ~P4_CCCR_OVF);
520
	}
521 522

	return overflow;
523 524
}

525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547
static void p4_pmu_disable_pebs(void)
{
	/*
	 * FIXME
	 *
	 * It's still allowed that two threads setup same cache
	 * events so we can't simply clear metrics until we knew
	 * noone is depending on us, so we need kind of counter
	 * for "ReplayEvent" users.
	 *
	 * What is more complex -- RAW events, if user (for some
	 * reason) will pass some cache event metric with improper
	 * event opcode -- it's fine from hardware point of view
	 * but completely nonsence from "meaning" of such action.
	 *
	 * So at moment let leave metrics turned on forever -- it's
	 * ok for now but need to be revisited!
	 *
	 * (void)checking_wrmsrl(MSR_IA32_PEBS_ENABLE, (u64)0);
	 * (void)checking_wrmsrl(MSR_P4_PEBS_MATRIX_VERT, (u64)0);
	 */
}

548 549 550 551 552 553 554 555 556 557 558
static inline void p4_pmu_disable_event(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	/*
	 * If event gets disabled while counter is in overflowed
	 * state we need to clear P4_CCCR_OVF, otherwise interrupt get
	 * asserted again and again
	 */
	(void)checking_wrmsrl(hwc->config_base + hwc->idx,
		(u64)(p4_config_unpack_cccr(hwc->config)) &
559
			~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
560 561 562 563 564 565 566
}

static void p4_pmu_disable_all(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int idx;

567
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
568 569 570 571 572
		struct perf_event *event = cpuc->events[idx];
		if (!test_bit(idx, cpuc->active_mask))
			continue;
		p4_pmu_disable_event(event);
	}
573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592

	p4_pmu_disable_pebs();
}

/* configuration must be valid */
static void p4_pmu_enable_pebs(u64 config)
{
	struct p4_pebs_bind *bind;
	unsigned int idx;

	BUILD_BUG_ON(P4_PEBS_METRIC__max > P4_PEBS_CONFIG_METRIC_MASK);

	idx = p4_config_unpack_metric(config);
	if (idx == P4_PEBS_METRIC__none)
		return;

	bind = &p4_pebs_bind_map[idx];

	(void)checking_wrmsrl(MSR_IA32_PEBS_ENABLE,	(u64)bind->metric_pebs);
	(void)checking_wrmsrl(MSR_P4_PEBS_MATRIX_VERT,	(u64)bind->metric_vert);
593 594 595 596 597 598 599
}

static void p4_pmu_enable_event(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;
	int thread = p4_ht_config_thread(hwc->config);
	u64 escr_conf = p4_config_unpack_escr(p4_clear_ht_bit(hwc->config));
600 601 602
	unsigned int idx = p4_config_unpack_event(hwc->config);
	struct p4_event_bind *bind;
	u64 escr_addr, cccr;
603

604 605
	bind = &p4_event_bind_map[idx];
	escr_addr = (u64)bind->escr_msr[thread];
606 607 608 609 610 611 612 613

	/*
	 * - we dont support cascaded counters yet
	 * - and counter 1 is broken (erratum)
	 */
	WARN_ON_ONCE(p4_is_event_cascaded(hwc->config));
	WARN_ON_ONCE(hwc->idx == 1);

614 615 616 617 618 619 620
	/* we need a real Event value */
	escr_conf &= ~P4_ESCR_EVENT_MASK;
	escr_conf |= P4_ESCR_EVENT(P4_OPCODE_EVNT(bind->opcode));

	cccr = p4_config_unpack_cccr(hwc->config);

	/*
621 622
	 * it could be Cache event so we need to write metrics
	 * into additional MSRs
623
	 */
624
	p4_pmu_enable_pebs(hwc->config);
625 626

	(void)checking_wrmsrl(escr_addr, escr_conf);
627
	(void)checking_wrmsrl(hwc->config_base + hwc->idx,
628
				(cccr & ~P4_CCCR_RESERVED) | P4_CCCR_ENABLE);
629 630
}

631
static void p4_pmu_enable_all(int added)
632 633 634 635
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	int idx;

636
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
		struct perf_event *event = cpuc->events[idx];
		if (!test_bit(idx, cpuc->active_mask))
			continue;
		p4_pmu_enable_event(event);
	}
}

static int p4_pmu_handle_irq(struct pt_regs *regs)
{
	struct perf_sample_data data;
	struct cpu_hw_events *cpuc;
	struct perf_event *event;
	struct hw_perf_event *hwc;
	int idx, handled = 0;
	u64 val;

	data.addr = 0;
	data.raw = NULL;

	cpuc = &__get_cpu_var(cpu_hw_events);

658
	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
659
		int overflow;
660 661 662 663 664 665 666 667 668

		if (!test_bit(idx, cpuc->active_mask))
			continue;

		event = cpuc->events[idx];
		hwc = &event->hw;

		WARN_ON_ONCE(hwc->idx != idx);

669
		/* it might be unflagged overflow */
670
		overflow = p4_pmu_clear_cccr_ovf(hwc);
671 672

		val = x86_perf_event_update(event);
673
		if (!overflow && (val & (1ULL << (x86_pmu.cntval_bits - 1))))
674 675
			continue;

676 677
		handled += overflow;

678 679
		/* event overflow for sure */
		data.period = event->hw.last_period;
680 681 682 683 684 685 686 687 688 689 690 691 692

		if (!x86_perf_event_set_period(event))
			continue;
		if (perf_event_overflow(event, 1, &data, regs))
			p4_pmu_disable_event(event);
	}

	if (handled) {
		/* p4 quirk: unmask it again */
		apic_write(APIC_LVTPC, apic_read(APIC_LVTPC) & ~APIC_LVT_MASKED);
		inc_irq_stat(apic_perf_irqs);
	}

693
	return handled > 0;
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
}

/*
 * swap thread specific fields according to a thread
 * we are going to run on
 */
static void p4_pmu_swap_config_ts(struct hw_perf_event *hwc, int cpu)
{
	u32 escr, cccr;

	/*
	 * we either lucky and continue on same cpu or no HT support
	 */
	if (!p4_should_swap_ts(hwc->config, cpu))
		return;

	/*
	 * the event is migrated from an another logical
	 * cpu, so we need to swap thread specific flags
	 */

	escr = p4_config_unpack_escr(hwc->config);
	cccr = p4_config_unpack_cccr(hwc->config);

	if (p4_ht_thread(cpu)) {
		cccr &= ~P4_CCCR_OVF_PMI_T0;
		cccr |= P4_CCCR_OVF_PMI_T1;
721 722 723
		if (escr & P4_ESCR_T0_OS) {
			escr &= ~P4_ESCR_T0_OS;
			escr |= P4_ESCR_T1_OS;
724
		}
725 726 727
		if (escr & P4_ESCR_T0_USR) {
			escr &= ~P4_ESCR_T0_USR;
			escr |= P4_ESCR_T1_USR;
728 729 730 731 732 733 734
		}
		hwc->config  = p4_config_pack_escr(escr);
		hwc->config |= p4_config_pack_cccr(cccr);
		hwc->config |= P4_CONFIG_HT;
	} else {
		cccr &= ~P4_CCCR_OVF_PMI_T1;
		cccr |= P4_CCCR_OVF_PMI_T0;
735 736 737
		if (escr & P4_ESCR_T1_OS) {
			escr &= ~P4_ESCR_T1_OS;
			escr |= P4_ESCR_T0_OS;
738
		}
739 740 741
		if (escr & P4_ESCR_T1_USR) {
			escr &= ~P4_ESCR_T1_USR;
			escr |= P4_ESCR_T0_USR;
742 743 744 745 746 747 748
		}
		hwc->config  = p4_config_pack_escr(escr);
		hwc->config |= p4_config_pack_cccr(cccr);
		hwc->config &= ~P4_CONFIG_HT;
	}
}

749 750
/*
 * ESCR address hashing is tricky, ESCRs are not sequential
751
 * in memory but all starts from MSR_P4_BSU_ESCR0 (0x03a0) and
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809
 * the metric between any ESCRs is laid in range [0xa0,0xe1]
 *
 * so we make ~70% filled hashtable
 */

#define P4_ESCR_MSR_BASE		0x000003a0
#define P4_ESCR_MSR_MAX			0x000003e1
#define P4_ESCR_MSR_TABLE_SIZE		(P4_ESCR_MSR_MAX - P4_ESCR_MSR_BASE + 1)
#define P4_ESCR_MSR_IDX(msr)		(msr - P4_ESCR_MSR_BASE)
#define P4_ESCR_MSR_TABLE_ENTRY(msr)	[P4_ESCR_MSR_IDX(msr)] = msr

static const unsigned int p4_escr_table[P4_ESCR_MSR_TABLE_SIZE] = {
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ALF_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ALF_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BPU_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BPU_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BSU_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BSU_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR2),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR3),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR4),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR5),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_DAC_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_DAC_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FIRM_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FIRM_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FLAME_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FLAME_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FSB_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FSB_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IQ_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IQ_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IS_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IS_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ITLB_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ITLB_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IX_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IX_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MOB_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MOB_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MS_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MS_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_PMH_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_PMH_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_RAT_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_RAT_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SAAT_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SAAT_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SSU_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SSU_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TBPU_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TBPU_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TC_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TC_ESCR1),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_U2L_ESCR0),
	P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_U2L_ESCR1),
810 811 812 813
};

static int p4_get_escr_idx(unsigned int addr)
{
814
	unsigned int idx = P4_ESCR_MSR_IDX(addr);
815

816 817 818
	if (unlikely(idx >= P4_ESCR_MSR_TABLE_SIZE	||
			!p4_escr_table[idx]		||
			p4_escr_table[idx] != addr)) {
819 820
		WARN_ONCE(1, "P4 PMU: Wrong address passed: %x\n", addr);
		return -1;
821 822
	}

823
	return idx;
824 825
}

826 827 828
static int p4_next_cntr(int thread, unsigned long *used_mask,
			struct p4_event_bind *bind)
{
829
	int i, j;
830 831

	for (i = 0; i < P4_CNTR_LIMIT; i++) {
832 833
		j = bind->cntr[thread][i];
		if (j != -1 && !test_bit(j, used_mask))
834 835 836 837 838 839
			return j;
	}

	return -1;
}

840 841 842
static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
{
	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
843
	unsigned long escr_mask[BITS_TO_LONGS(P4_ESCR_MSR_TABLE_SIZE)];
844
	int cpu = smp_processor_id();
845 846 847 848
	struct hw_perf_event *hwc;
	struct p4_event_bind *bind;
	unsigned int i, thread, num;
	int cntr_idx, escr_idx;
849 850

	bitmap_zero(used_mask, X86_PMC_IDX_MAX);
851
	bitmap_zero(escr_mask, P4_ESCR_MSR_TABLE_SIZE);
852 853

	for (i = 0, num = n; i < n; i++, num--) {
854

855 856
		hwc = &cpuc->event_list[i]->hw;
		thread = p4_ht_thread(cpu);
857 858
		bind = p4_config_get_bind(hwc->config);
		escr_idx = p4_get_escr_idx(bind->escr_msr[thread]);
859 860
		if (unlikely(escr_idx == -1))
			goto done;
861 862

		if (hwc->idx != -1 && !p4_should_swap_ts(hwc->config, cpu)) {
863
			cntr_idx = hwc->idx;
864 865 866 867 868
			if (assign)
				assign[i] = hwc->idx;
			goto reserve;
		}

869 870
		cntr_idx = p4_next_cntr(thread, used_mask, bind);
		if (cntr_idx == -1 || test_bit(escr_idx, escr_mask))
871 872 873
			goto done;

		p4_pmu_swap_config_ts(hwc, cpu);
874 875
		if (assign)
			assign[i] = cntr_idx;
876
reserve:
877
		set_bit(cntr_idx, used_mask);
878 879 880 881 882 883 884
		set_bit(escr_idx, escr_mask);
	}

done:
	return num ? -ENOSPC : 0;
}

885
static __initconst const struct x86_pmu p4_pmu = {
886 887 888 889 890 891 892 893 894
	.name			= "Netburst P4/Xeon",
	.handle_irq		= p4_pmu_handle_irq,
	.disable_all		= p4_pmu_disable_all,
	.enable_all		= p4_pmu_enable_all,
	.enable			= p4_pmu_enable_event,
	.disable		= p4_pmu_disable_event,
	.eventsel		= MSR_P4_BPU_CCCR0,
	.perfctr		= MSR_P4_BPU_PERFCTR0,
	.event_map		= p4_pmu_event_map,
895
	.max_events		= ARRAY_SIZE(p4_general_events),
896 897 898 899 900 901 902
	.get_event_constraints	= x86_get_event_constraints,
	/*
	 * IF HT disabled we may need to use all
	 * ARCH_P4_MAX_CCCR counters simulaneously
	 * though leave it restricted at moment assuming
	 * HT is on
	 */
903
	.num_counters		= ARCH_P4_MAX_CCCR,
904
	.apic			= 1,
905 906
	.cntval_bits		= 40,
	.cntval_mask		= (1ULL << 40) - 1,
907 908 909
	.max_period		= (1ULL << 39) - 1,
	.hw_config		= p4_hw_config,
	.schedule_events	= p4_pmu_schedule_events,
910 911 912 913 914 915 916 917 918
	/*
	 * This handles erratum N15 in intel doc 249199-029,
	 * the counter may not be updated correctly on write
	 * so we need a second write operation to do the trick
	 * (the official workaround didn't work)
	 *
	 * the former idea is taken from OProfile code
	 */
	.perfctr_second_write	= 1,
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
};

static __init int p4_pmu_init(void)
{
	unsigned int low, high;

	/* If we get stripped -- indexig fails */
	BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC);

	rdmsr(MSR_IA32_MISC_ENABLE, low, high);
	if (!(low & (1 << 7))) {
		pr_cont("unsupported Netburst CPU model %d ",
			boot_cpu_data.x86_model);
		return -ENODEV;
	}

935
	memcpy(hw_cache_event_ids, p4_hw_cache_event_ids,
936
		sizeof(hw_cache_event_ids));
937

938 939 940 941 942 943 944 945
	pr_cont("Netburst events, ");

	x86_pmu = p4_pmu;

	return 0;
}

#endif /* CONFIG_CPU_SUP_INTEL */