aspm.c 25.3 KB
Newer Older
S
Shaohua Li 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
/*
 * File:	drivers/pci/pcie/aspm.c
 * Enabling PCIE link L0s/L1 state and Clock Power Management
 *
 * Copyright (C) 2007 Intel
 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/pci_regs.h>
#include <linux/errno.h>
#include <linux/pm.h>
#include <linux/init.h>
#include <linux/slab.h>
19
#include <linux/jiffies.h>
20
#include <linux/delay.h>
S
Shaohua Li 已提交
21 22 23 24 25 26 27 28
#include <linux/pci-aspm.h>
#include "../pci.h"

#ifdef MODULE_PARAM_PREFIX
#undef MODULE_PARAM_PREFIX
#endif
#define MODULE_PARAM_PREFIX "pcie_aspm."

29 30 31
struct aspm_latency {
	u32 l0s;			/* L0s latency (nsec) */
	u32 l1;				/* L1 latency (nsec) */
S
Shaohua Li 已提交
32 33 34
};

struct pcie_link_state {
35 36 37 38 39
	struct pci_dev *pdev;		/* Upstream component of the Link */
	struct pcie_link_state *parent;	/* pointer to the parent Link state */
	struct list_head sibling;	/* node in link_list */
	struct list_head children;	/* list of child link states */
	struct list_head link;		/* node in parent's children list */
S
Shaohua Li 已提交
40 41

	/* ASPM state */
42 43 44 45
	u32 aspm_support:2;		/* Supported ASPM state */
	u32 aspm_enabled:2;		/* Enabled ASPM state */
	u32 aspm_default:2;		/* Default ASPM state by BIOS */

46 47 48 49 50
	/* Clock PM state */
	u32 clkpm_capable:1;		/* Clock PM capable? */
	u32 clkpm_enabled:1;		/* Current Clock PM state */
	u32 clkpm_default:1;		/* Default Clock PM state by BIOS */

51 52
	/* Latencies */
	struct aspm_latency latency;	/* Exit latency */
S
Shaohua Li 已提交
53
	/*
54 55
	 * Endpoint acceptable latencies. A pcie downstream port only
	 * has one slot under it, so at most there are 8 functions.
S
Shaohua Li 已提交
56
	 */
57
	struct aspm_latency acceptable[8];
S
Shaohua Li 已提交
58 59
};

60
static int aspm_disabled, aspm_force;
S
Shaohua Li 已提交
61 62 63 64 65 66 67 68 69 70 71 72 73
static DEFINE_MUTEX(aspm_lock);
static LIST_HEAD(link_list);

#define POLICY_DEFAULT 0	/* BIOS default setting */
#define POLICY_PERFORMANCE 1	/* high performance */
#define POLICY_POWERSAVE 2	/* high power saving */
static int aspm_policy;
static const char *policy_str[] = {
	[POLICY_DEFAULT] = "default",
	[POLICY_PERFORMANCE] = "performance",
	[POLICY_POWERSAVE] = "powersave"
};

74 75
#define LINK_RETRAIN_TIMEOUT HZ

76
static int policy_to_aspm_state(struct pcie_link_state *link)
S
Shaohua Li 已提交
77 78 79 80 81 82 83
{
	switch (aspm_policy) {
	case POLICY_PERFORMANCE:
		/* Disable ASPM and Clock PM */
		return 0;
	case POLICY_POWERSAVE:
		/* Enable ASPM L0s/L1 */
84
		return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
S
Shaohua Li 已提交
85
	case POLICY_DEFAULT:
86
		return link->aspm_default;
S
Shaohua Li 已提交
87 88 89 90
	}
	return 0;
}

91
static int policy_to_clkpm_state(struct pcie_link_state *link)
S
Shaohua Li 已提交
92 93 94 95 96 97 98 99 100
{
	switch (aspm_policy) {
	case POLICY_PERFORMANCE:
		/* Disable ASPM and Clock PM */
		return 0;
	case POLICY_POWERSAVE:
		/* Disable Clock PM */
		return 1;
	case POLICY_DEFAULT:
101
		return link->clkpm_default;
S
Shaohua Li 已提交
102 103 104 105
	}
	return 0;
}

K
Kenji Kaneshige 已提交
106
static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
S
Shaohua Li 已提交
107 108 109
{
	int pos;
	u16 reg16;
110 111
	struct pci_dev *child;
	struct pci_bus *linkbus = link->pdev->subordinate;
S
Shaohua Li 已提交
112

113 114
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
S
Shaohua Li 已提交
115 116
		if (!pos)
			return;
117
		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
S
Shaohua Li 已提交
118 119 120 121
		if (enable)
			reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
		else
			reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
122
		pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
S
Shaohua Li 已提交
123
	}
124
	link->clkpm_enabled = !!enable;
S
Shaohua Li 已提交
125 126
}

K
Kenji Kaneshige 已提交
127 128 129 130 131 132 133 134 135 136 137
static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
{
	/* Don't enable Clock PM if the link is not Clock PM capable */
	if (!link->clkpm_capable && enable)
		return;
	/* Need nothing if the specified equals to current state */
	if (link->clkpm_enabled == enable)
		return;
	pcie_set_clkpm_nocheck(link, enable);
}

138
static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
S
Shaohua Li 已提交
139
{
140
	int pos, capable = 1, enabled = 1;
S
Shaohua Li 已提交
141 142
	u32 reg32;
	u16 reg16;
143 144
	struct pci_dev *child;
	struct pci_bus *linkbus = link->pdev->subordinate;
S
Shaohua Li 已提交
145 146

	/* All functions should have the same cap and state, take the worst */
147 148
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
S
Shaohua Li 已提交
149 150
		if (!pos)
			return;
151
		pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, &reg32);
S
Shaohua Li 已提交
152 153 154 155 156
		if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
			capable = 0;
			enabled = 0;
			break;
		}
157
		pci_read_config_word(child, pos + PCI_EXP_LNKCTL, &reg16);
S
Shaohua Li 已提交
158 159 160
		if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
			enabled = 0;
	}
161 162
	link->clkpm_enabled = enabled;
	link->clkpm_default = enabled;
163
	link->clkpm_capable = (blacklist) ? 0 : capable;
164 165
}

166
static bool pcie_aspm_downstream_has_switch(struct pcie_link_state *link)
167
{
168 169
	struct pci_dev *child;
	struct pci_bus *linkbus = link->pdev->subordinate;
170

171 172
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		if (child->pcie_type == PCI_EXP_TYPE_UPSTREAM)
173 174 175
			return true;
	}
	return false;
S
Shaohua Li 已提交
176 177 178 179 180 181 182
}

/*
 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
 *   could use common clock. If they are, configure them to use the
 *   common clock. That will reduce the ASPM state exit latency.
 */
183
static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
S
Shaohua Li 已提交
184
{
185 186
	int ppos, cpos, same_clock = 1;
	u16 reg16, parent_reg, child_reg[8];
187
	unsigned long start_jiffies;
188 189
	struct pci_dev *child, *parent = link->pdev;
	struct pci_bus *linkbus = parent->subordinate;
S
Shaohua Li 已提交
190
	/*
191
	 * All functions of a slot should have the same Slot Clock
S
Shaohua Li 已提交
192
	 * Configuration, so just check one function
193 194 195
	 */
	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
	BUG_ON(!child->is_pcie);
S
Shaohua Li 已提交
196 197

	/* Check downstream component if bit Slot Clock Configuration is 1 */
198 199
	cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
	pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, &reg16);
S
Shaohua Li 已提交
200 201 202 203
	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
		same_clock = 0;

	/* Check upstream component if bit Slot Clock Configuration is 1 */
204 205
	ppos = pci_find_capability(parent, PCI_CAP_ID_EXP);
	pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
S
Shaohua Li 已提交
206 207 208 209
	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
		same_clock = 0;

	/* Configure downstream component, all functions */
210 211 212 213
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
		pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, &reg16);
		child_reg[PCI_FUNC(child->devfn)] = reg16;
S
Shaohua Li 已提交
214 215 216 217
		if (same_clock)
			reg16 |= PCI_EXP_LNKCTL_CCC;
		else
			reg16 &= ~PCI_EXP_LNKCTL_CCC;
218
		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
S
Shaohua Li 已提交
219 220 221
	}

	/* Configure upstream component */
222
	pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, &reg16);
223
	parent_reg = reg16;
S
Shaohua Li 已提交
224 225 226 227
	if (same_clock)
		reg16 |= PCI_EXP_LNKCTL_CCC;
	else
		reg16 &= ~PCI_EXP_LNKCTL_CCC;
228
	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
S
Shaohua Li 已提交
229

230
	/* Retrain link */
S
Shaohua Li 已提交
231
	reg16 |= PCI_EXP_LNKCTL_RL;
232
	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
S
Shaohua Li 已提交
233

234
	/* Wait for link training end. Break out after waiting for timeout */
235
	start_jiffies = jiffies;
236
	for (;;) {
237
		pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, &reg16);
S
Shaohua Li 已提交
238 239
		if (!(reg16 & PCI_EXP_LNKSTA_LT))
			break;
240 241 242
		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
			break;
		msleep(1);
S
Shaohua Li 已提交
243
	}
244 245 246 247 248 249 250 251 252 253
	if (!(reg16 & PCI_EXP_LNKSTA_LT))
		return;

	/* Training failed. Restore common clock configurations */
	dev_printk(KERN_ERR, &parent->dev,
		   "ASPM: Could not configure common clock\n");
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		cpos = pci_find_capability(child, PCI_CAP_ID_EXP);
		pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
				      child_reg[PCI_FUNC(child->devfn)]);
254
	}
255
	pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
S
Shaohua Li 已提交
256 257
}

258 259
/* Convert L0s latency encoding to ns */
static u32 calc_l0s_latency(u32 encoding)
S
Shaohua Li 已提交
260
{
261 262 263 264
	if (encoding == 0x7)
		return (5 * 1000);	/* > 4us */
	return (64 << encoding);
}
S
Shaohua Li 已提交
265

266 267 268 269 270 271
/* Convert L0s acceptable latency encoding to ns */
static u32 calc_l0s_acceptable(u32 encoding)
{
	if (encoding == 0x7)
		return -1U;
	return (64 << encoding);
S
Shaohua Li 已提交
272 273
}

274 275
/* Convert L1 latency encoding to ns */
static u32 calc_l1_latency(u32 encoding)
S
Shaohua Li 已提交
276
{
277 278 279 280
	if (encoding == 0x7)
		return (65 * 1000);	/* > 64us */
	return (1000 << encoding);
}
S
Shaohua Li 已提交
281

282 283 284 285 286 287
/* Convert L1 acceptable latency encoding to ns */
static u32 calc_l1_acceptable(u32 encoding)
{
	if (encoding == 0x7)
		return -1U;
	return (1000 << encoding);
S
Shaohua Li 已提交
288 289 290
}

static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
291
				     u32 *l0s, u32 *l1, u32 *enabled)
S
Shaohua Li 已提交
292 293 294
{
	int pos;
	u16 reg16;
295
	u32 reg32, encoding;
S
Shaohua Li 已提交
296

297
	*l0s = *l1 = *enabled = 0;
S
Shaohua Li 已提交
298 299 300 301
	pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
	pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, &reg32);
	*state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
	if (*state != PCIE_LINK_STATE_L0S &&
302
	    *state != (PCIE_LINK_STATE_L1 | PCIE_LINK_STATE_L0S))
S
Shaohua Li 已提交
303 304 305 306
		*state = 0;
	if (*state == 0)
		return;

307 308
	encoding = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
	*l0s = calc_l0s_latency(encoding);
S
Shaohua Li 已提交
309
	if (*state & PCIE_LINK_STATE_L1) {
310 311
		encoding = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
		*l1 = calc_l1_latency(encoding);
S
Shaohua Li 已提交
312 313
	}
	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
314
	*enabled = reg16 & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
S
Shaohua Li 已提交
315 316
}

317
static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
S
Shaohua Li 已提交
318
{
319
	u32 support, l0s, l1, enabled;
320 321
	struct pci_dev *child, *parent = link->pdev;
	struct pci_bus *linkbus = parent->subordinate;
S
Shaohua Li 已提交
322

323 324 325 326 327 328 329 330 331 332 333
	if (blacklist) {
		/* Set support state to 0, so we will disable ASPM later */
		link->aspm_support = 0;
		link->aspm_default = 0;
		link->aspm_enabled = PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1;
		return;
	}

	/* Configure common clock before checking latencies */
	pcie_aspm_configure_common_clock(link);

S
Shaohua Li 已提交
334
	/* upstream component states */
335 336 337 338 339
	pcie_aspm_get_cap_device(parent, &support, &l0s, &l1, &enabled);
	link->aspm_support = support;
	link->latency.l0s = l0s;
	link->latency.l1 = l1;
	link->aspm_enabled = enabled;
340

S
Shaohua Li 已提交
341
	/* downstream component states, all functions have the same setting */
342 343 344 345 346 347 348
	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
	pcie_aspm_get_cap_device(child, &support, &l0s, &l1, &enabled);
	link->aspm_support &= support;
	link->latency.l0s = max_t(u32, link->latency.l0s, l0s);
	link->latency.l1 = max_t(u32, link->latency.l1, l1);

	if (!link->aspm_support)
S
Shaohua Li 已提交
349
		return;
350

351 352
	link->aspm_enabled &= link->aspm_support;
	link->aspm_default = link->aspm_enabled;
S
Shaohua Li 已提交
353 354

	/* ENDPOINT states*/
355
	list_for_each_entry(child, &linkbus->devices, bus_list) {
S
Shaohua Li 已提交
356
		int pos;
357
		u32 reg32, encoding;
358
		struct aspm_latency *acceptable =
359
			&link->acceptable[PCI_FUNC(child->devfn)];
S
Shaohua Li 已提交
360

361 362
		if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
		    child->pcie_type != PCI_EXP_TYPE_LEG_END)
S
Shaohua Li 已提交
363 364
			continue;

365 366
		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
367 368
		encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
		acceptable->l0s = calc_l0s_acceptable(encoding);
369
		if (link->aspm_support & PCIE_LINK_STATE_L1) {
370 371
			encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
			acceptable->l1 = calc_l1_acceptable(encoding);
S
Shaohua Li 已提交
372 373 374 375
		}
	}
}

376 377 378 379 380 381 382 383 384 385 386
/**
 * __pcie_aspm_check_state_one - check latency for endpoint device.
 * @endpoint: pointer to the struct pci_dev of endpoint device
 *
 * TBD: The latency from the endpoint to root complex vary per switch's
 * upstream link state above the device. Here we just do a simple check
 * which assumes all links above the device can be in L1 state, that
 * is we just consider the worst case. If switch's upstream link can't
 * be put into L0S/L1, then our check is too strictly.
 */
static u32 __pcie_aspm_check_state_one(struct pci_dev *endpoint, u32 state)
S
Shaohua Li 已提交
387
{
388
	u32 l1_switch_latency = 0;
389
	struct aspm_latency *acceptable;
390
	struct pcie_link_state *link;
S
Shaohua Li 已提交
391

392 393 394
	link = endpoint->bus->self->link_state;
	state &= link->aspm_support;
	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
S
Shaohua Li 已提交
395

396
	while (link && state) {
397
		if ((state & PCIE_LINK_STATE_L0S) &&
398
		    (link->latency.l0s > acceptable->l0s))
399 400
			state &= ~PCIE_LINK_STATE_L0S;
		if ((state & PCIE_LINK_STATE_L1) &&
401
		    (link->latency.l1 + l1_switch_latency > acceptable->l1))
402
			state &= ~PCIE_LINK_STATE_L1;
403 404 405 406 407 408
		link = link->parent;
		/*
		 * Every switch on the path to root complex need 1
		 * more microsecond for L1. Spec doesn't mention L0s.
		 */
		l1_switch_latency += 1000;
S
Shaohua Li 已提交
409 410 411 412
	}
	return state;
}

413
static u32 pcie_aspm_check_state(struct pcie_link_state *link, u32 state)
S
Shaohua Li 已提交
414
{
415 416 417
	pci_power_t power_state;
	struct pci_dev *child;
	struct pci_bus *linkbus = link->pdev->subordinate;
S
Shaohua Li 已提交
418

419
	/* If no child, ignore the link */
420
	if (list_empty(&linkbus->devices))
421
		return state;
422 423 424 425 426 427 428 429 430 431 432

	list_for_each_entry(child, &linkbus->devices, bus_list) {
		/*
		 * If downstream component of a link is pci bridge, we
		 * disable ASPM for now for the link
		 */
		if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
			return 0;

		if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
		     child->pcie_type != PCI_EXP_TYPE_LEG_END))
S
Shaohua Li 已提交
433 434
			continue;
		/* Device not in D0 doesn't need check latency */
435 436 437
		power_state = child->current_state;
		if (power_state == PCI_D1 || power_state == PCI_D2 ||
		    power_state == PCI_D3hot || power_state == PCI_D3cold)
S
Shaohua Li 已提交
438
			continue;
439
		state = __pcie_aspm_check_state_one(child, state);
S
Shaohua Li 已提交
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
	}
	return state;
}

static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
{
	u16 reg16;
	int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);

	pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, &reg16);
	reg16 &= ~0x3;
	reg16 |= state;
	pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
}

455
static void __pcie_aspm_config_link(struct pcie_link_state *link, u32 state)
S
Shaohua Li 已提交
456
{
457 458
	struct pci_dev *child, *parent = link->pdev;
	struct pci_bus *linkbus = parent->subordinate;
S
Shaohua Li 已提交
459

460
	/* If no child, disable the link */
461
	if (list_empty(&linkbus->devices))
462
		state = 0;
S
Shaohua Li 已提交
463
	/*
464 465
	 * If the downstream component has pci bridge function, don't
	 * do ASPM now.
S
Shaohua Li 已提交
466
	 */
467 468 469
	list_for_each_entry(child, &linkbus->devices, bus_list) {
		if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
			return;
S
Shaohua Li 已提交
470 471
	}
	/*
472 473 474 475
	 * Spec 2.0 suggests all functions should be configured the
	 * same setting for ASPM. Enabling ASPM L1 should be done in
	 * upstream component first and then downstream, and vice
	 * versa for disabling ASPM L1. Spec doesn't mention L0S.
S
Shaohua Li 已提交
476 477
	 */
	if (state & PCIE_LINK_STATE_L1)
478
		__pcie_aspm_config_one_dev(parent, state);
S
Shaohua Li 已提交
479

480 481
	list_for_each_entry(child, &linkbus->devices, bus_list)
		__pcie_aspm_config_one_dev(child, state);
S
Shaohua Li 已提交
482 483

	if (!(state & PCIE_LINK_STATE_L1))
484
		__pcie_aspm_config_one_dev(parent, state);
S
Shaohua Li 已提交
485

486
	link->aspm_enabled = state;
S
Shaohua Li 已提交
487 488
}

489 490 491 492 493 494 495 496
static struct pcie_link_state *get_root_port_link(struct pcie_link_state *link)
{
	struct pcie_link_state *root_port_link = link;
	while (root_port_link->parent)
		root_port_link = root_port_link->parent;
	return root_port_link;
}

497 498 499
/* Check the whole hierarchy, and configure each link in the hierarchy */
static void __pcie_aspm_configure_link_state(struct pcie_link_state *link,
					     u32 state)
S
Shaohua Li 已提交
500
{
501
	struct pcie_link_state *leaf, *root = get_root_port_link(link);
S
Shaohua Li 已提交
502

503
	state &= (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
S
Shaohua Li 已提交
504

505
	/* Check all links who have specific root port link */
506
	list_for_each_entry(leaf, &link_list, sibling) {
507
		if (!list_empty(&leaf->children) ||
508
		    get_root_port_link(leaf) != root)
509
			continue;
510
		state = pcie_aspm_check_state(leaf, state);
511
	}
512 513 514
	/* Check root port link too in case it hasn't children */
	state = pcie_aspm_check_state(root, state);
	if (link->aspm_enabled == state)
S
Shaohua Li 已提交
515
		return;
516
	/*
517
	 * We must change the hierarchy. See comments in
518 519 520
	 * __pcie_aspm_config_link for the order
	 **/
	if (state & PCIE_LINK_STATE_L1) {
521
		list_for_each_entry(leaf, &link_list, sibling) {
522 523
			if (get_root_port_link(leaf) == root)
				__pcie_aspm_config_link(leaf, state);
524 525
		}
	} else {
526
		list_for_each_entry_reverse(leaf, &link_list, sibling) {
527 528
			if (get_root_port_link(leaf) == root)
				__pcie_aspm_config_link(leaf, state);
529 530
		}
	}
S
Shaohua Li 已提交
531 532 533 534 535 536
}

/*
 * pcie_aspm_configure_link_state: enable/disable PCI express link state
 * @pdev: the root port or switch downstream port
 */
537 538
static void pcie_aspm_configure_link_state(struct pcie_link_state *link,
					   u32 state)
S
Shaohua Li 已提交
539 540 541
{
	down_read(&pci_bus_sem);
	mutex_lock(&aspm_lock);
542
	__pcie_aspm_configure_link_state(link, state);
S
Shaohua Li 已提交
543 544 545 546
	mutex_unlock(&aspm_lock);
	up_read(&pci_bus_sem);
}

547
static void free_link_state(struct pcie_link_state *link)
S
Shaohua Li 已提交
548
{
549 550
	link->pdev->link_state = NULL;
	kfree(link);
S
Shaohua Li 已提交
551 552
}

553 554
static int pcie_aspm_sanity_check(struct pci_dev *pdev)
{
555 556
	struct pci_dev *child;
	int pos;
557
	u32 reg32;
558
	/*
559 560
	 * Some functions in a slot might not all be PCIE functions,
	 * very strange. Disable ASPM for the whole slot
561
	 */
562 563 564
	list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
		pos = pci_find_capability(child, PCI_CAP_ID_EXP);
		if (!pos)
565
			return -EINVAL;
566 567 568 569
		/*
		 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
		 * RBER bit to determine if a function is 1.1 version device
		 */
570
		pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, &reg32);
S
Sitsofe Wheeler 已提交
571
		if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
572
			dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
573 574
				" on pre-1.1 PCIe device.  You can enable it"
				" with 'pcie_aspm=force'\n");
575 576
			return -EINVAL;
		}
577 578 579 580
	}
	return 0;
}

581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
static struct pcie_link_state *pcie_aspm_setup_link_state(struct pci_dev *pdev)
{
	struct pcie_link_state *link;
	int blacklist = !!pcie_aspm_sanity_check(pdev);

	link = kzalloc(sizeof(*link), GFP_KERNEL);
	if (!link)
		return NULL;
	INIT_LIST_HEAD(&link->sibling);
	INIT_LIST_HEAD(&link->children);
	INIT_LIST_HEAD(&link->link);
	link->pdev = pdev;
	if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
		struct pcie_link_state *parent;
		parent = pdev->bus->parent->self->link_state;
		if (!parent) {
			kfree(link);
			return NULL;
		}
		link->parent = parent;
		list_add(&link->link, &parent->children);
	}
	list_add(&link->sibling, &link_list);

	pdev->link_state = link;

	/* Check ASPM capability */
	pcie_aspm_cap_init(link, blacklist);

	/* Check Clock PM capability */
	pcie_clkpm_cap_init(link, blacklist);

	return link;
}

S
Shaohua Li 已提交
616 617 618 619 620 621 622
/*
 * pcie_aspm_init_link_state: Initiate PCI express link state.
 * It is called after the pcie and its children devices are scaned.
 * @pdev: the root port or switch downstream port
 */
void pcie_aspm_init_link_state(struct pci_dev *pdev)
{
623 624
	u32 state;
	struct pcie_link_state *link;
S
Shaohua Li 已提交
625 626 627 628

	if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
		return;
	if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
629
	    pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
S
Shaohua Li 已提交
630
		return;
631

632 633
	/* VIA has a strange chipset, root port is under a bridge */
	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
634
	    pdev->bus->self)
635
		return;
636

S
Shaohua Li 已提交
637 638 639 640 641
	down_read(&pci_bus_sem);
	if (list_empty(&pdev->subordinate->devices))
		goto out;

	mutex_lock(&aspm_lock);
642 643 644 645 646 647 648 649 650 651
	link = pcie_aspm_setup_link_state(pdev);
	if (!link)
		goto unlock;
	/*
	 * Setup initial ASPM state
	 *
	 * If link has switch, delay the link config. The leaf link
	 * initialization will config the whole hierarchy. But we must
	 * make sure BIOS doesn't set unsupported link state.
	 */
652
	if (pcie_aspm_downstream_has_switch(link)) {
653 654
		state = pcie_aspm_check_state(link, link->aspm_default);
		__pcie_aspm_config_link(link, state);
655
	} else {
656 657
		state = policy_to_aspm_state(link);
		__pcie_aspm_configure_link_state(link, state);
658
	}
S
Shaohua Li 已提交
659

660 661
	/* Setup initial Clock PM state */
	state = (link->clkpm_capable) ? policy_to_clkpm_state(link) : 0;
K
Kenji Kaneshige 已提交
662
	pcie_set_clkpm(link, state);
663
unlock:
S
Shaohua Li 已提交
664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
	mutex_unlock(&aspm_lock);
out:
	up_read(&pci_bus_sem);
}

/* @pdev: the endpoint device */
void pcie_aspm_exit_link_state(struct pci_dev *pdev)
{
	struct pci_dev *parent = pdev->bus->self;
	struct pcie_link_state *link_state = parent->link_state;

	if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
		return;
	if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
		parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
		return;
	down_read(&pci_bus_sem);
	mutex_lock(&aspm_lock);

	/*
	 * All PCIe functions are in one slot, remove one function will remove
685
	 * the whole slot, so just wait until we are the last function left.
S
Shaohua Li 已提交
686
	 */
687
	if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
S
Shaohua Li 已提交
688 689 690 691
		goto out;

	/* All functions are removed, so just disable ASPM for the link */
	__pcie_aspm_config_one_dev(parent, 0);
692
	list_del(&link_state->sibling);
693
	list_del(&link_state->link);
S
Shaohua Li 已提交
694 695
	/* Clock PM is for endpoint device */

696
	free_link_state(link_state);
S
Shaohua Li 已提交
697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
out:
	mutex_unlock(&aspm_lock);
	up_read(&pci_bus_sem);
}

/* @pdev: the root port or switch downstream port */
void pcie_aspm_pm_state_change(struct pci_dev *pdev)
{
	struct pcie_link_state *link_state = pdev->link_state;

	if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
		return;
	if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
		pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
		return;
	/*
	 * devices changed PM state, we should recheck if latency meets all
	 * functions' requirement
	 */
716
	pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
S
Shaohua Li 已提交
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
}

/*
 * pci_disable_link_state - disable pci device's link state, so the link will
 * never enter specific states
 */
void pci_disable_link_state(struct pci_dev *pdev, int state)
{
	struct pci_dev *parent = pdev->bus->self;
	struct pcie_link_state *link_state;

	if (aspm_disabled || !pdev->is_pcie)
		return;
	if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
	    pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
		parent = pdev;
	if (!parent || !parent->link_state)
		return;

	down_read(&pci_bus_sem);
	mutex_lock(&aspm_lock);
	link_state = parent->link_state;
739
	link_state->aspm_support &= ~state;
740
	__pcie_aspm_configure_link_state(link_state, link_state->aspm_enabled);
K
Kenji Kaneshige 已提交
741 742 743 744
	if (state & PCIE_LINK_STATE_CLKPM) {
		link_state->clkpm_capable = 0;
		pcie_set_clkpm(link_state, 0);
	}
S
Shaohua Li 已提交
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765
	mutex_unlock(&aspm_lock);
	up_read(&pci_bus_sem);
}
EXPORT_SYMBOL(pci_disable_link_state);

static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
{
	int i;
	struct pcie_link_state *link_state;

	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
		if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
			break;
	if (i >= ARRAY_SIZE(policy_str))
		return -EINVAL;
	if (i == aspm_policy)
		return 0;

	down_read(&pci_bus_sem);
	mutex_lock(&aspm_lock);
	aspm_policy = i;
766
	list_for_each_entry(link_state, &link_list, sibling) {
767 768
		__pcie_aspm_configure_link_state(link_state,
			policy_to_aspm_state(link_state));
K
Kenji Kaneshige 已提交
769
		pcie_set_clkpm(link_state, policy_to_clkpm_state(link_state));
S
Shaohua Li 已提交
770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
	}
	mutex_unlock(&aspm_lock);
	up_read(&pci_bus_sem);
	return 0;
}

static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
{
	int i, cnt = 0;
	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
		if (i == aspm_policy)
			cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
		else
			cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
	return cnt;
}

module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
	NULL, 0644);

#ifdef CONFIG_PCIEASPM_DEBUG
static ssize_t link_state_show(struct device *dev,
		struct device_attribute *attr,
		char *buf)
{
	struct pci_dev *pci_device = to_pci_dev(dev);
	struct pcie_link_state *link_state = pci_device->link_state;

798
	return sprintf(buf, "%d\n", link_state->aspm_enabled);
S
Shaohua Li 已提交
799 800 801 802 803 804 805
}

static ssize_t link_state_store(struct device *dev,
		struct device_attribute *attr,
		const char *buf,
		size_t n)
{
806
	struct pci_dev *pdev = to_pci_dev(dev);
S
Shaohua Li 已提交
807 808 809 810 811 812 813
	int state;

	if (n < 1)
		return -EINVAL;
	state = buf[0]-'0';
	if (state >= 0 && state <= 3) {
		/* setup link aspm state */
814
		pcie_aspm_configure_link_state(pdev->link_state, state);
S
Shaohua Li 已提交
815 816 817 818 819 820 821 822 823 824 825 826 827
		return n;
	}

	return -EINVAL;
}

static ssize_t clk_ctl_show(struct device *dev,
		struct device_attribute *attr,
		char *buf)
{
	struct pci_dev *pci_device = to_pci_dev(dev);
	struct pcie_link_state *link_state = pci_device->link_state;

828
	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
S
Shaohua Li 已提交
829 830 831 832 833 834 835
}

static ssize_t clk_ctl_store(struct device *dev,
		struct device_attribute *attr,
		const char *buf,
		size_t n)
{
K
Kenji Kaneshige 已提交
836
	struct pci_dev *pdev = to_pci_dev(dev);
S
Shaohua Li 已提交
837 838 839 840 841 842 843 844
	int state;

	if (n < 1)
		return -EINVAL;
	state = buf[0]-'0';

	down_read(&pci_bus_sem);
	mutex_lock(&aspm_lock);
K
Kenji Kaneshige 已提交
845
	pcie_set_clkpm_nocheck(pdev->link_state, !!state);
S
Shaohua Li 已提交
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863
	mutex_unlock(&aspm_lock);
	up_read(&pci_bus_sem);

	return n;
}

static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);

static char power_group[] = "power";
void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
{
	struct pcie_link_state *link_state = pdev->link_state;

	if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
		pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
		return;

864
	if (link_state->aspm_support)
S
Shaohua Li 已提交
865 866
		sysfs_add_file_to_group(&pdev->dev.kobj,
			&dev_attr_link_state.attr, power_group);
867
	if (link_state->clkpm_capable)
S
Shaohua Li 已提交
868 869 870 871 872 873 874 875 876 877 878 879
		sysfs_add_file_to_group(&pdev->dev.kobj,
			&dev_attr_clk_ctl.attr, power_group);
}

void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
{
	struct pcie_link_state *link_state = pdev->link_state;

	if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
		pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
		return;

880
	if (link_state->aspm_support)
S
Shaohua Li 已提交
881 882
		sysfs_remove_file_from_group(&pdev->dev.kobj,
			&dev_attr_link_state.attr, power_group);
883
	if (link_state->clkpm_capable)
S
Shaohua Li 已提交
884 885 886 887 888 889 890
		sysfs_remove_file_from_group(&pdev->dev.kobj,
			&dev_attr_clk_ctl.attr, power_group);
}
#endif

static int __init pcie_aspm_disable(char *str)
{
891 892 893 894 895 896 897
	if (!strcmp(str, "off")) {
		aspm_disabled = 1;
		printk(KERN_INFO "PCIe ASPM is disabled\n");
	} else if (!strcmp(str, "force")) {
		aspm_force = 1;
		printk(KERN_INFO "PCIe ASPM is forcedly enabled\n");
	}
S
Shaohua Li 已提交
898 899 900
	return 1;
}

901
__setup("pcie_aspm=", pcie_aspm_disable);
S
Shaohua Li 已提交
902

903 904
void pcie_no_aspm(void)
{
905 906
	if (!aspm_force)
		aspm_disabled = 1;
907 908
}

909 910 911 912 913 914 915
/**
 * pcie_aspm_enabled - is PCIe ASPM enabled?
 *
 * Returns true if ASPM has not been disabled by the command-line option
 * pcie_aspm=off.
 **/
int pcie_aspm_enabled(void)
S
Shaohua Li 已提交
916
{
917
       return !aspm_disabled;
S
Shaohua Li 已提交
918
}
919
EXPORT_SYMBOL(pcie_aspm_enabled);
S
Shaohua Li 已提交
920