idle.c 22.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
/*
 * PowerNV cpuidle code
 *
 * Copyright 2015 IBM Corp.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/types.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/of.h>
16 17
#include <linux/device.h>
#include <linux/cpu.h>
18 19

#include <asm/firmware.h>
20
#include <asm/machdep.h>
21 22 23 24 25
#include <asm/opal.h>
#include <asm/cputhreads.h>
#include <asm/cpuidle.h>
#include <asm/code-patching.h>
#include <asm/smp.h>
26
#include <asm/runlatch.h>
27
#include <asm/dbell.h>
28 29 30 31

#include "powernv.h"
#include "subcore.h"

32 33 34
/* Power ISA 3.0 allows for stop states 0x0 - 0xF */
#define MAX_STOP_STATE	0xF

35 36 37
#define P9_STOP_SPR_MSR 2000
#define P9_STOP_SPR_PSSCR      855

38
static u32 supported_cpuidle_states;
39 40
struct pnv_idle_states_t *pnv_idle_states;
int nr_pnv_idle_states;
41

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
/*
 * The default stop state that will be used by ppc_md.power_save
 * function on platforms that support stop instruction.
 */
static u64 pnv_default_stop_val;
static u64 pnv_default_stop_mask;
static bool default_stop_found;

/*
 * First deep stop state. Used to figure out when to save/restore
 * hypervisor context.
 */
u64 pnv_first_deep_stop_state = MAX_STOP_STATE;

/*
 * psscr value and mask of the deepest stop idle state.
 * Used when a cpu is offlined.
 */
static u64 pnv_deepest_stop_psscr_val;
static u64 pnv_deepest_stop_psscr_mask;
62
static u64 pnv_deepest_stop_flag;
63 64
static bool deepest_stop_found;

65
static int pnv_save_sprs_for_deep_states(void)
66 67 68 69 70
{
	int cpu;
	int rc;

	/*
71
	 * hid0, hid1, hid4, hid5, hmeer and lpcr values are symmetric across
72
	 * all cpus at boot. Get these reg values of current cpu and use the
73
	 * same across all cpus.
74
	 */
75
	uint64_t lpcr_val = mfspr(SPRN_LPCR);
76 77 78 79 80
	uint64_t hid0_val = mfspr(SPRN_HID0);
	uint64_t hid1_val = mfspr(SPRN_HID1);
	uint64_t hid4_val = mfspr(SPRN_HID4);
	uint64_t hid5_val = mfspr(SPRN_HID5);
	uint64_t hmeer_val = mfspr(SPRN_HMEER);
81 82
	uint64_t msr_val = MSR_IDLE;
	uint64_t psscr_val = pnv_deepest_stop_psscr_val;
83

84
	for_each_present_cpu(cpu) {
85
		uint64_t pir = get_hard_smp_processor_id(cpu);
86
		uint64_t hsprg0_val = (uint64_t)paca_ptrs[cpu];
87 88 89 90 91 92 93 94 95

		rc = opal_slw_set_reg(pir, SPRN_HSPRG0, hsprg0_val);
		if (rc != 0)
			return rc;

		rc = opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
		if (rc != 0)
			return rc;

96 97 98 99 100 101 102 103 104 105 106 107
		if (cpu_has_feature(CPU_FTR_ARCH_300)) {
			rc = opal_slw_set_reg(pir, P9_STOP_SPR_MSR, msr_val);
			if (rc)
				return rc;

			rc = opal_slw_set_reg(pir,
					      P9_STOP_SPR_PSSCR, psscr_val);

			if (rc)
				return rc;
		}

108 109 110 111 112 113 114 115 116 117 118
		/* HIDs are per core registers */
		if (cpu_thread_in_core(cpu) == 0) {

			rc = opal_slw_set_reg(pir, SPRN_HMEER, hmeer_val);
			if (rc != 0)
				return rc;

			rc = opal_slw_set_reg(pir, SPRN_HID0, hid0_val);
			if (rc != 0)
				return rc;

119 120
			/* Only p8 needs to set extra HID regiters */
			if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
121

122 123 124
				rc = opal_slw_set_reg(pir, SPRN_HID1, hid1_val);
				if (rc != 0)
					return rc;
125

126 127 128 129 130 131 132 133
				rc = opal_slw_set_reg(pir, SPRN_HID4, hid4_val);
				if (rc != 0)
					return rc;

				rc = opal_slw_set_reg(pir, SPRN_HID5, hid5_val);
				if (rc != 0)
					return rc;
			}
134 135 136 137 138 139 140 141 142 143 144 145 146
		}
	}

	return 0;
}

static void pnv_alloc_idle_core_states(void)
{
	int i, j;
	int nr_cores = cpu_nr_cores();
	u32 *core_idle_state;

	/*
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164
	 * core_idle_state - The lower 8 bits track the idle state of
	 * each thread of the core.
	 *
	 * The most significant bit is the lock bit.
	 *
	 * Initially all the bits corresponding to threads_per_core
	 * are set. They are cleared when the thread enters deep idle
	 * state like sleep and winkle/stop.
	 *
	 * Initially the lock bit is cleared.  The lock bit has 2
	 * purposes:
	 * 	a. While the first thread in the core waking up from
	 * 	   idle is restoring core state, it prevents other
	 * 	   threads in the core from switching to process
	 * 	   context.
	 * 	b. While the last thread in the core is saving the
	 *	   core state, it prevents a different thread from
	 *	   waking up.
165 166 167 168
	 */
	for (i = 0; i < nr_cores; i++) {
		int first_cpu = i * threads_per_core;
		int node = cpu_to_node(first_cpu);
169
		size_t paca_ptr_array_size;
170 171

		core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
172
		*core_idle_state = (1 << threads_per_core) - 1;
173 174
		paca_ptr_array_size = (threads_per_core *
				       sizeof(struct paca_struct *));
175 176 177 178

		for (j = 0; j < threads_per_core; j++) {
			int cpu = first_cpu + j;

179 180 181
			paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state;
			paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING;
			paca_ptrs[cpu]->thread_mask = 1 << j;
182 183 184 185 186
		}
	}

	update_subcore_sibling_mask();

187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
		int rc = pnv_save_sprs_for_deep_states();

		if (likely(!rc))
			return;

		/*
		 * The stop-api is unable to restore hypervisor
		 * resources on wakeup from platform idle states which
		 * lose full context. So disable such states.
		 */
		supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
		pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
		pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");

		if (cpu_has_feature(CPU_FTR_ARCH_300) &&
		    (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
			/*
			 * Use the default stop state for CPU-Hotplug
			 * if available.
			 */
			if (default_stop_found) {
				pnv_deepest_stop_psscr_val =
					pnv_default_stop_val;
				pnv_deepest_stop_psscr_mask =
					pnv_default_stop_mask;
				pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
					pnv_deepest_stop_psscr_val);
			} else { /* Fallback to snooze loop for CPU-Hotplug */
				deepest_stop_found = false;
				pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
			}
		}
	}
221 222 223 224 225 226 227 228
}

u32 pnv_get_supported_cpuidle_states(void)
{
	return supported_cpuidle_states;
}
EXPORT_SYMBOL_GPL(pnv_get_supported_cpuidle_states);

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
static void pnv_fastsleep_workaround_apply(void *info)

{
	int rc;
	int *err = info;

	rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
					OPAL_CONFIG_IDLE_APPLY);
	if (rc)
		*err = 1;
}

/*
 * Used to store fastsleep workaround state
 * 0 - Workaround applied/undone at fastsleep entry/exit path (Default)
 * 1 - Workaround applied once, never undone.
 */
static u8 fastsleep_workaround_applyonce;

static ssize_t show_fastsleep_workaround_applyonce(struct device *dev,
		struct device_attribute *attr, char *buf)
{
	return sprintf(buf, "%u\n", fastsleep_workaround_applyonce);
}

static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
		struct device_attribute *attr, const char *buf,
		size_t count)
{
	cpumask_t primary_thread_mask;
	int err;
	u8 val;

	if (kstrtou8(buf, 0, &val) || val != 1)
		return -EINVAL;

	if (fastsleep_workaround_applyonce == 1)
		return count;

	/*
	 * fastsleep_workaround_applyonce = 1 implies
	 * fastsleep workaround needs to be left in 'applied' state on all
	 * the cores. Do this by-
	 * 1. Patching out the call to 'undo' workaround in fastsleep exit path
273
	 * 2. Sending ipi to all the cores which have at least one online thread
274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
	 * 3. Patching out the call to 'apply' workaround in fastsleep entry
	 * path
	 * There is no need to send ipi to cores which have all threads
	 * offlined, as last thread of the core entering fastsleep or deeper
	 * state would have applied workaround.
	 */
	err = patch_instruction(
		(unsigned int *)pnv_fastsleep_workaround_at_exit,
		PPC_INST_NOP);
	if (err) {
		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit");
		goto fail;
	}

	get_online_cpus();
	primary_thread_mask = cpu_online_cores_map();
	on_each_cpu_mask(&primary_thread_mask,
				pnv_fastsleep_workaround_apply,
				&err, 1);
	put_online_cpus();
	if (err) {
		pr_err("fastsleep_workaround_applyonce change failed while running pnv_fastsleep_workaround_apply");
		goto fail;
	}

	err = patch_instruction(
		(unsigned int *)pnv_fastsleep_workaround_at_entry,
		PPC_INST_NOP);
	if (err) {
		pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry");
		goto fail;
	}

	fastsleep_workaround_applyonce = 1;

	return count;
fail:
	return -EIO;
}

static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
			show_fastsleep_workaround_applyonce,
			store_fastsleep_workaround_applyonce);

318 319 320 321 322 323 324
static unsigned long __power7_idle_type(unsigned long type)
{
	unsigned long srr1;

	if (!prep_irq_for_idle_irqsoff())
		return 0;

325
	__ppc64_runlatch_off();
326
	srr1 = power7_idle_insn(type);
327
	__ppc64_runlatch_on();
328 329 330 331 332 333 334 335

	fini_irq_for_idle_irqsoff();

	return srr1;
}

void power7_idle_type(unsigned long type)
{
336 337 338 339
	unsigned long srr1;

	srr1 = __power7_idle_type(type);
	irq_set_pending_from_srr1(srr1);
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
}

void power7_idle(void)
{
	if (!powersave_nap)
		return;

	power7_idle_type(PNV_THREAD_NAP);
}

static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
				      unsigned long stop_psscr_mask)
{
	unsigned long psscr;
	unsigned long srr1;

	if (!prep_irq_for_idle_irqsoff())
		return 0;

	psscr = mfspr(SPRN_PSSCR);
	psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;

362
	__ppc64_runlatch_off();
363
	srr1 = power9_idle_stop(psscr);
364
	__ppc64_runlatch_on();
365 366 367 368 369 370 371 372 373

	fini_irq_for_idle_irqsoff();

	return srr1;
}

void power9_idle_type(unsigned long stop_psscr_val,
				      unsigned long stop_psscr_mask)
{
374 375 376 377
	unsigned long srr1;

	srr1 = __power9_idle_type(stop_psscr_val, stop_psscr_mask);
	irq_set_pending_from_srr1(srr1);
378 379
}

380 381 382
/*
 * Used for ppc_md.power_save which needs a function with no parameters
 */
383
void power9_idle(void)
384
{
385
	power9_idle_type(pnv_default_stop_val, pnv_default_stop_mask);
386
}
387

388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
 * This is used in working around bugs in thread reconfiguration
 * on POWER9 (at least up to Nimbus DD2.2) relating to transactional
 * memory and the way that XER[SO] is checkpointed.
 * This function forces the core into SMT4 in order by asking
 * all other threads not to stop, and sending a message to any
 * that are in a stop state.
 * Must be called with preemption disabled.
 */
void pnv_power9_force_smt4_catch(void)
{
	int cpu, cpu0, thr;
	int awake_threads = 1;		/* this thread is awake */
	int poke_threads = 0;
	int need_awake = threads_per_core;

	cpu = smp_processor_id();
	cpu0 = cpu & ~(threads_per_core - 1);
	for (thr = 0; thr < threads_per_core; ++thr) {
		if (cpu != cpu0 + thr)
409
			atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
410 411 412 413
	}
	/* order setting dont_stop vs testing requested_psscr */
	mb();
	for (thr = 0; thr < threads_per_core; ++thr) {
414
		if (!paca_ptrs[cpu0+thr]->requested_psscr)
415 416 417 418 419 420 421 422 423 424 425 426
			++awake_threads;
		else
			poke_threads |= (1 << thr);
	}

	/* If at least 3 threads are awake, the core is in SMT4 already */
	if (awake_threads < need_awake) {
		/* We have to wake some threads; we'll use msgsnd */
		for (thr = 0; thr < threads_per_core; ++thr) {
			if (poke_threads & (1 << thr)) {
				ppc_msgsnd_sync();
				ppc_msgsnd(PPC_DBELL_MSGTYPE, 0,
427
					   paca_ptrs[cpu0+thr]->hw_cpu_id);
428 429 430 431 432 433
			}
		}
		/* now spin until at least 3 threads are awake */
		do {
			for (thr = 0; thr < threads_per_core; ++thr) {
				if ((poke_threads & (1 << thr)) &&
434
				    !paca_ptrs[cpu0+thr]->requested_psscr) {
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
					++awake_threads;
					poke_threads &= ~(1 << thr);
				}
			}
		} while (awake_threads < need_awake);
	}
}
EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_catch);

void pnv_power9_force_smt4_release(void)
{
	int cpu, cpu0, thr;

	cpu = smp_processor_id();
	cpu0 = cpu & ~(threads_per_core - 1);

	/* clear all the dont_stop flags */
	for (thr = 0; thr < threads_per_core; ++thr) {
		if (cpu != cpu0 + thr)
454
			atomic_dec(&paca_ptrs[cpu0+thr]->dont_stop);
455 456 457 458 459
	}
}
EXPORT_SYMBOL_GPL(pnv_power9_force_smt4_release);
#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */

460
#ifdef CONFIG_HOTPLUG_CPU
461 462

void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val)
463 464 465 466
{
	u64 pir = get_hard_smp_processor_id(cpu);

	mtspr(SPRN_LPCR, lpcr_val);
467 468 469 470 471 472 473

	/*
	 * Program the LPCR via stop-api only if the deepest stop state
	 * can lose hypervisor context.
	 */
	if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT)
		opal_slw_set_reg(pir, SPRN_LPCR, lpcr_val);
474 475
}

476 477 478
/*
 * pnv_cpu_offline: A function that puts the CPU into the deepest
 * available platform idle state on a CPU-Offline.
479
 * interrupts hard disabled and no lazy irq pending.
480 481 482 483 484 485
 */
unsigned long pnv_cpu_offline(unsigned int cpu)
{
	unsigned long srr1;
	u32 idle_states = pnv_get_supported_cpuidle_states();

486
	__ppc64_runlatch_off();
487

488
	if (cpu_has_feature(CPU_FTR_ARCH_300) && deepest_stop_found) {
489 490 491 492 493
		unsigned long psscr;

		psscr = mfspr(SPRN_PSSCR);
		psscr = (psscr & ~pnv_deepest_stop_psscr_mask) |
						pnv_deepest_stop_psscr_val;
494
		srr1 = power9_offline_stop(psscr);
495

496 497
	} else if ((idle_states & OPAL_PM_WINKLE_ENABLED) &&
		   (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) {
498
		srr1 = power7_idle_insn(PNV_THREAD_WINKLE);
499 500
	} else if ((idle_states & OPAL_PM_SLEEP_ENABLED) ||
		   (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
501
		srr1 = power7_idle_insn(PNV_THREAD_SLEEP);
502
	} else if (idle_states & OPAL_PM_NAP_ENABLED) {
503
		srr1 = power7_idle_insn(PNV_THREAD_NAP);
504 505 506 507 508 509 510 511
	} else {
		/* This is the fallback method. We emulate snooze */
		while (!generic_check_cpu_restart(cpu)) {
			HMT_low();
			HMT_very_low();
		}
		srr1 = 0;
		HMT_medium();
512 513
	}

514
	__ppc64_runlatch_on();
515

516 517
	return srr1;
}
518
#endif
519

520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
/*
 * Power ISA 3.0 idle initialization.
 *
 * POWER ISA 3.0 defines a new SPR Processor stop Status and Control
 * Register (PSSCR) to control idle behavior.
 *
 * PSSCR layout:
 * ----------------------------------------------------------
 * | PLS | /// | SD | ESL | EC | PSLL | /// | TR | MTL | RL |
 * ----------------------------------------------------------
 * 0      4     41   42    43   44     48    54   56    60
 *
 * PSSCR key fields:
 *	Bits 0:3  - Power-Saving Level Status (PLS). This field indicates the
 *	lowest power-saving state the thread entered since stop instruction was
 *	last executed.
 *
 *	Bit 41 - Status Disable(SD)
 *	0 - Shows PLS entries
 *	1 - PLS entries are all 0
 *
 *	Bit 42 - Enable State Loss
 *	0 - No state is lost irrespective of other fields
 *	1 - Allows state loss
 *
 *	Bit 43 - Exit Criterion
 *	0 - Exit from power-save mode on any interrupt
 *	1 - Exit from power-save mode controlled by LPCR's PECE bits
 *
 *	Bits 44:47 - Power-Saving Level Limit
 *	This limits the power-saving level that can be entered into.
 *
 *	Bits 60:63 - Requested Level
 *	Used to specify which power-saving level must be entered on executing
 *	stop instruction
555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
 */

int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags)
{
	int err = 0;

	/*
	 * psscr_mask == 0xf indicates an older firmware.
	 * Set remaining fields of psscr to the default values.
	 * See NOTE above definition of PSSCR_HV_DEFAULT_VAL
	 */
	if (*psscr_mask == 0xf) {
		*psscr_val = *psscr_val | PSSCR_HV_DEFAULT_VAL;
		*psscr_mask = PSSCR_HV_DEFAULT_MASK;
		return err;
	}

	/*
	 * New firmware is expected to set the psscr_val bits correctly.
	 * Validate that the following invariants are correctly maintained by
	 * the new firmware.
	 * - ESL bit value matches the EC bit value.
	 * - ESL bit is set for all the deep stop states.
	 */
	if (GET_PSSCR_ESL(*psscr_val) != GET_PSSCR_EC(*psscr_val)) {
		err = ERR_EC_ESL_MISMATCH;
	} else if ((flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
		GET_PSSCR_ESL(*psscr_val) == 0) {
		err = ERR_DEEP_STATE_ESL_MISMATCH;
	}

	return err;
}

/*
 * pnv_arch300_idle_init: Initializes the default idle state, first
 *                        deep idle state and deepest idle state on
 *                        ISA 3.0 CPUs.
593 594 595 596 597 598
 *
 * @np: /ibm,opal/power-mgt device node
 * @flags: cpu-idle-state-flags array
 * @dt_idle_states: Number of idle state entries
 * Returns 0 on success
 */
599
static int __init pnv_power9_idle_init(void)
600
{
601
	u64 max_residency_ns = 0;
602
	int i;
603

604
	/*
605 606 607
	 * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask},
	 * and the pnv_default_stop_{val,mask}.
	 *
608 609
	 * pnv_first_deep_stop_state should be set to the first stop
	 * level to cause hypervisor state loss.
610 611 612 613 614 615
	 *
	 * pnv_deepest_stop_{val,mask} should be set to values corresponding to
	 * the deepest stop state.
	 *
	 * pnv_default_stop_{val,mask} should be set to values corresponding to
	 * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state.
616 617
	 */
	pnv_first_deep_stop_state = MAX_STOP_STATE;
618
	for (i = 0; i < nr_pnv_idle_states; i++) {
619
		int err;
620 621
		struct pnv_idle_states_t *state = &pnv_idle_states[i];
		u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK;
622

623 624
		if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
		    pnv_first_deep_stop_state > psscr_rl)
625
			pnv_first_deep_stop_state = psscr_rl;
626

627 628 629
		err = validate_psscr_val_mask(&state->psscr_val,
					      &state->psscr_mask,
					      state->flags);
630
		if (err) {
631
			report_invalid_psscr_val(state->psscr_val, err);
632 633 634
			continue;
		}

635 636
		state->valid = true;

637 638 639 640 641
		if (max_residency_ns < state->residency_ns) {
			max_residency_ns = state->residency_ns;
			pnv_deepest_stop_psscr_val = state->psscr_val;
			pnv_deepest_stop_psscr_mask = state->psscr_mask;
			pnv_deepest_stop_flag = state->flags;
642 643 644 645
			deepest_stop_found = true;
		}

		if (!default_stop_found &&
646 647 648
		    (state->flags & OPAL_PM_STOP_INST_FAST)) {
			pnv_default_stop_val = state->psscr_val;
			pnv_default_stop_mask = state->psscr_mask;
649 650 651 652
			default_stop_found = true;
		}
	}

653 654 655 656 657
	if (unlikely(!default_stop_found)) {
		pr_warn("cpuidle-powernv: No suitable default stop state found. Disabling platform idle.\n");
	} else {
		ppc_md.power_save = power9_idle;
		pr_info("cpuidle-powernv: Default stop: psscr = 0x%016llx,mask=0x%016llx\n",
658 659 660
			pnv_default_stop_val, pnv_default_stop_mask);
	}

661 662 663 664
	if (unlikely(!deepest_stop_found)) {
		pr_warn("cpuidle-powernv: No suitable stop state for CPU-Hotplug. Offlined CPUs will busy wait");
	} else {
		pr_info("cpuidle-powernv: Deepest stop: psscr = 0x%016llx,mask=0x%016llx\n",
665 666
			pnv_deepest_stop_psscr_val,
			pnv_deepest_stop_psscr_mask);
667 668
	}

669 670
	pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n",
		pnv_first_deep_stop_state);
671 672

	return 0;
673 674 675 676 677 678 679 680 681
}

/*
 * Probe device tree for supported idle states
 */
static void __init pnv_probe_idle_states(void)
{
	int i;

682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
	if (nr_pnv_idle_states < 0) {
		pr_warn("cpuidle-powernv: no idle states found in the DT\n");
		return;
	}

	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
		if (pnv_power9_idle_init())
			return;
	}

	for (i = 0; i < nr_pnv_idle_states; i++)
		supported_cpuidle_states |= pnv_idle_states[i].flags;
}

/*
 * This function parses device-tree and populates all the information
 * into pnv_idle_states structure. It also sets up nr_pnv_idle_states
 * which is the number of cpuidle states discovered through device-tree.
 */

static int pnv_parse_cpuidle_dt(void)
{
	struct device_node *np;
	int nr_idle_states, i;
	int rc = 0;
	u32 *temp_u32;
	u64 *temp_u64;
	const char **temp_string;

711 712
	np = of_find_node_by_path("/ibm,opal/power-mgt");
	if (!np) {
713
		pr_warn("opal: PowerMgmt Node not found\n");
714
		return -ENODEV;
715
	}
716 717 718 719 720 721 722 723 724 725 726 727
	nr_idle_states = of_property_count_u32_elems(np,
						"ibm,cpu-idle-state-flags");

	pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states),
				  GFP_KERNEL);
	temp_u32 = kcalloc(nr_idle_states, sizeof(u32),  GFP_KERNEL);
	temp_u64 = kcalloc(nr_idle_states, sizeof(u64),  GFP_KERNEL);
	temp_string = kcalloc(nr_idle_states, sizeof(char *),  GFP_KERNEL);

	if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) {
		pr_err("Could not allocate memory for dt parsing\n");
		rc = -ENOMEM;
728 729 730
		goto out;
	}

731 732 733
	/* Read flags */
	if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags",
				       temp_u32, nr_idle_states)) {
734
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n");
735
		rc = -EINVAL;
736 737
		goto out;
	}
738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
	for (i = 0; i < nr_idle_states; i++)
		pnv_idle_states[i].flags = temp_u32[i];

	/* Read latencies */
	if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns",
				       temp_u32, nr_idle_states)) {
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
		rc = -EINVAL;
		goto out;
	}
	for (i = 0; i < nr_idle_states; i++)
		pnv_idle_states[i].latency_ns = temp_u32[i];

	/* Read residencies */
	if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns",
				       temp_u32, nr_idle_states)) {
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
		rc = -EINVAL;
		goto out;
	}
	for (i = 0; i < nr_idle_states; i++)
		pnv_idle_states[i].residency_ns = temp_u32[i];
760

761
	/* For power9 */
762
	if (cpu_has_feature(CPU_FTR_ARCH_300)) {
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777
		/* Read pm_crtl_val */
		if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr",
					       temp_u64, nr_idle_states)) {
			pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n");
			rc = -EINVAL;
			goto out;
		}
		for (i = 0; i < nr_idle_states; i++)
			pnv_idle_states[i].psscr_val = temp_u64[i];

		/* Read pm_crtl_mask */
		if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask",
					       temp_u64, nr_idle_states)) {
			pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n");
			rc = -EINVAL;
778
			goto out;
779 780 781
		}
		for (i = 0; i < nr_idle_states; i++)
			pnv_idle_states[i].psscr_mask = temp_u64[i];
782 783
	}

784 785 786 787 788
	/*
	 * power8 specific properties ibm,cpu-idle-state-pmicr-mask and
	 * ibm,cpu-idle-state-pmicr-val were never used and there is no
	 * plan to use it in near future. Hence, not parsing these properties
	 */
789

790 791 792 793 794 795 796
	if (of_property_read_string_array(np, "ibm,cpu-idle-state-names",
					  temp_string, nr_idle_states) < 0) {
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
		rc = -EINVAL;
		goto out;
	}
	for (i = 0; i < nr_idle_states; i++)
797
		strlcpy(pnv_idle_states[i].name, temp_string[i],
798 799 800
			PNV_IDLE_NAME_LEN);
	nr_pnv_idle_states = nr_idle_states;
	rc = 0;
801
out:
802 803 804 805
	kfree(temp_u32);
	kfree(temp_u64);
	kfree(temp_string);
	return rc;
806
}
807

808 809
static int __init pnv_init_idle_states(void)
{
810
	int rc = 0;
811 812
	supported_cpuidle_states = 0;

813 814
	/* In case we error out nr_pnv_idle_states will be zero */
	nr_pnv_idle_states = 0;
815 816
	if (cpuidle_disable != IDLE_NO_OVERRIDE)
		goto out;
817 818 819
	rc = pnv_parse_cpuidle_dt();
	if (rc)
		return rc;
820 821
	pnv_probe_idle_states();

822 823 824 825 826 827 828
	if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
		patch_instruction(
			(unsigned int *)pnv_fastsleep_workaround_at_entry,
			PPC_INST_NOP);
		patch_instruction(
			(unsigned int *)pnv_fastsleep_workaround_at_exit,
			PPC_INST_NOP);
829 830 831 832 833 834 835 836
	} else {
		/*
		 * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
		 * workaround is needed to use fastsleep. Provide sysfs
		 * control to choose how this workaround has to be applied.
		 */
		device_create_file(cpu_subsys.dev_root,
				&dev_attr_fastsleep_workaround_applyonce);
837
	}
838

839
	pnv_alloc_idle_core_states();
840 841 842

	if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
		ppc_md.power_save = power7_idle;
843

844 845 846
out:
	return 0;
}
847
machine_subsys_initcall(powernv, pnv_init_idle_states);