1 /*
2 * File: drivers/pci/pcie/aspm.c
3 * Enabling PCIe link L0s/L1 state and Clock Power Management
4 *
5 * Copyright (C) 2007 Intel
6 * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7 * Copyright (C) Shaohua Li (shaohua.li@intel.com)
8 */
9
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/moduleparam.h>
13 #include <linux/pci.h>
14 #include <linux/pci_regs.h>
15 #include <linux/errno.h>
16 #include <linux/pm.h>
17 #include <linux/init.h>
18 #include <linux/slab.h>
19 #include <linux/jiffies.h>
20 #include <linux/delay.h>
21 #include <linux/pci-aspm.h>
22 #include "../pci.h"
23
24 #ifdef MODULE_PARAM_PREFIX
25 #undef MODULE_PARAM_PREFIX
26 #endif
27 #define MODULE_PARAM_PREFIX "pcie_aspm."
28
29 /* Note: those are not register definitions */
30 #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
31 #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
32 #define ASPM_STATE_L1 (4) /* L1 state */
33 #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
34 #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1)
35
36 struct aspm_latency {
37 u32 l0s; /* L0s latency (nsec) */
38 u32 l1; /* L1 latency (nsec) */
39 };
40
41 struct pcie_link_state {
42 struct pci_dev *pdev; /* Upstream component of the Link */
43 struct pcie_link_state *root; /* pointer to the root port link */
44 struct pcie_link_state *parent; /* pointer to the parent Link state */
45 struct list_head sibling; /* node in link_list */
46 struct list_head children; /* list of child link states */
47 struct list_head link; /* node in parent's children list */
48
49 /* ASPM state */
50 u32 aspm_support:3; /* Supported ASPM state */
51 u32 aspm_enabled:3; /* Enabled ASPM state */
52 u32 aspm_capable:3; /* Capable ASPM state with latency */
53 u32 aspm_default:3; /* Default ASPM state by BIOS */
54 u32 aspm_disable:3; /* Disabled ASPM state */
55
56 /* Clock PM state */
57 u32 clkpm_capable:1; /* Clock PM capable? */
58 u32 clkpm_enabled:1; /* Current Clock PM state */
59 u32 clkpm_default:1; /* Default Clock PM state by BIOS */
60
61 /* Exit latencies */
62 struct aspm_latency latency_up; /* Upstream direction exit latency */
63 struct aspm_latency latency_dw; /* Downstream direction exit latency */
64 /*
65 * Endpoint acceptable latencies. A pcie downstream port only
66 * has one slot under it, so at most there are 8 functions.
67 */
68 struct aspm_latency acceptable[8];
69 };
70
71 static int aspm_disabled, aspm_force;
72 static bool aspm_support_enabled = true;
73 static DEFINE_MUTEX(aspm_lock);
74 static LIST_HEAD(link_list);
75
76 #define POLICY_DEFAULT 0 /* BIOS default setting */
77 #define POLICY_PERFORMANCE 1 /* high performance */
78 #define POLICY_POWERSAVE 2 /* high power saving */
79
80 #ifdef CONFIG_PCIEASPM_PERFORMANCE
81 static int aspm_policy = POLICY_PERFORMANCE;
82 #elif defined CONFIG_PCIEASPM_POWERSAVE
83 static int aspm_policy = POLICY_POWERSAVE;
84 #else
85 static int aspm_policy;
86 #endif
87
88 static const char *policy_str[] = {
89 [POLICY_DEFAULT] = "default",
90 [POLICY_PERFORMANCE] = "performance",
91 [POLICY_POWERSAVE] = "powersave"
92 };
93
94 #define LINK_RETRAIN_TIMEOUT HZ
95
policy_to_aspm_state(struct pcie_link_state * link)96 static int policy_to_aspm_state(struct pcie_link_state *link)
97 {
98 switch (aspm_policy) {
99 case POLICY_PERFORMANCE:
100 /* Disable ASPM and Clock PM */
101 return 0;
102 case POLICY_POWERSAVE:
103 /* Enable ASPM L0s/L1 */
104 return ASPM_STATE_ALL;
105 case POLICY_DEFAULT:
106 return link->aspm_default;
107 }
108 return 0;
109 }
110
policy_to_clkpm_state(struct pcie_link_state * link)111 static int policy_to_clkpm_state(struct pcie_link_state *link)
112 {
113 switch (aspm_policy) {
114 case POLICY_PERFORMANCE:
115 /* Disable ASPM and Clock PM */
116 return 0;
117 case POLICY_POWERSAVE:
118 /* Disable Clock PM */
119 return 1;
120 case POLICY_DEFAULT:
121 return link->clkpm_default;
122 }
123 return 0;
124 }
125
pcie_set_clkpm_nocheck(struct pcie_link_state * link,int enable)126 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
127 {
128 int pos;
129 u16 reg16;
130 struct pci_dev *child;
131 struct pci_bus *linkbus = link->pdev->subordinate;
132
133 list_for_each_entry(child, &linkbus->devices, bus_list) {
134 pos = pci_pcie_cap(child);
135 if (!pos)
136 return;
137 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
138 if (enable)
139 reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
140 else
141 reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
142 pci_write_config_word(child, pos + PCI_EXP_LNKCTL, reg16);
143 }
144 link->clkpm_enabled = !!enable;
145 }
146
pcie_set_clkpm(struct pcie_link_state * link,int enable)147 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
148 {
149 /* Don't enable Clock PM if the link is not Clock PM capable */
150 if (!link->clkpm_capable && enable)
151 enable = 0;
152 /* Need nothing if the specified equals to current state */
153 if (link->clkpm_enabled == enable)
154 return;
155 pcie_set_clkpm_nocheck(link, enable);
156 }
157
pcie_clkpm_cap_init(struct pcie_link_state * link,int blacklist)158 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
159 {
160 int pos, capable = 1, enabled = 1;
161 u32 reg32;
162 u16 reg16;
163 struct pci_dev *child;
164 struct pci_bus *linkbus = link->pdev->subordinate;
165
166 /* All functions should have the same cap and state, take the worst */
167 list_for_each_entry(child, &linkbus->devices, bus_list) {
168 pos = pci_pcie_cap(child);
169 if (!pos)
170 return;
171 pci_read_config_dword(child, pos + PCI_EXP_LNKCAP, ®32);
172 if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
173 capable = 0;
174 enabled = 0;
175 break;
176 }
177 pci_read_config_word(child, pos + PCI_EXP_LNKCTL, ®16);
178 if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
179 enabled = 0;
180 }
181 link->clkpm_enabled = enabled;
182 link->clkpm_default = enabled;
183 link->clkpm_capable = (blacklist) ? 0 : capable;
184 }
185
186 /*
187 * pcie_aspm_configure_common_clock: check if the 2 ends of a link
188 * could use common clock. If they are, configure them to use the
189 * common clock. That will reduce the ASPM state exit latency.
190 */
pcie_aspm_configure_common_clock(struct pcie_link_state * link)191 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
192 {
193 int ppos, cpos, same_clock = 1;
194 u16 reg16, parent_reg, child_reg[8];
195 unsigned long start_jiffies;
196 struct pci_dev *child, *parent = link->pdev;
197 struct pci_bus *linkbus = parent->subordinate;
198 /*
199 * All functions of a slot should have the same Slot Clock
200 * Configuration, so just check one function
201 */
202 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
203 BUG_ON(!pci_is_pcie(child));
204
205 /* Check downstream component if bit Slot Clock Configuration is 1 */
206 cpos = pci_pcie_cap(child);
207 pci_read_config_word(child, cpos + PCI_EXP_LNKSTA, ®16);
208 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
209 same_clock = 0;
210
211 /* Check upstream component if bit Slot Clock Configuration is 1 */
212 ppos = pci_pcie_cap(parent);
213 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
214 if (!(reg16 & PCI_EXP_LNKSTA_SLC))
215 same_clock = 0;
216
217 /* Configure downstream component, all functions */
218 list_for_each_entry(child, &linkbus->devices, bus_list) {
219 cpos = pci_pcie_cap(child);
220 pci_read_config_word(child, cpos + PCI_EXP_LNKCTL, ®16);
221 child_reg[PCI_FUNC(child->devfn)] = reg16;
222 if (same_clock)
223 reg16 |= PCI_EXP_LNKCTL_CCC;
224 else
225 reg16 &= ~PCI_EXP_LNKCTL_CCC;
226 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL, reg16);
227 }
228
229 /* Configure upstream component */
230 pci_read_config_word(parent, ppos + PCI_EXP_LNKCTL, ®16);
231 parent_reg = reg16;
232 if (same_clock)
233 reg16 |= PCI_EXP_LNKCTL_CCC;
234 else
235 reg16 &= ~PCI_EXP_LNKCTL_CCC;
236 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
237
238 /* Retrain link */
239 reg16 |= PCI_EXP_LNKCTL_RL;
240 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, reg16);
241
242 /* Wait for link training end. Break out after waiting for timeout */
243 start_jiffies = jiffies;
244 for (;;) {
245 pci_read_config_word(parent, ppos + PCI_EXP_LNKSTA, ®16);
246 if (!(reg16 & PCI_EXP_LNKSTA_LT))
247 break;
248 if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
249 break;
250 msleep(1);
251 }
252 if (!(reg16 & PCI_EXP_LNKSTA_LT))
253 return;
254
255 /* Training failed. Restore common clock configurations */
256 dev_printk(KERN_ERR, &parent->dev,
257 "ASPM: Could not configure common clock\n");
258 list_for_each_entry(child, &linkbus->devices, bus_list) {
259 cpos = pci_pcie_cap(child);
260 pci_write_config_word(child, cpos + PCI_EXP_LNKCTL,
261 child_reg[PCI_FUNC(child->devfn)]);
262 }
263 pci_write_config_word(parent, ppos + PCI_EXP_LNKCTL, parent_reg);
264 }
265
266 /* Convert L0s latency encoding to ns */
calc_l0s_latency(u32 encoding)267 static u32 calc_l0s_latency(u32 encoding)
268 {
269 if (encoding == 0x7)
270 return (5 * 1000); /* > 4us */
271 return (64 << encoding);
272 }
273
274 /* Convert L0s acceptable latency encoding to ns */
calc_l0s_acceptable(u32 encoding)275 static u32 calc_l0s_acceptable(u32 encoding)
276 {
277 if (encoding == 0x7)
278 return -1U;
279 return (64 << encoding);
280 }
281
282 /* Convert L1 latency encoding to ns */
calc_l1_latency(u32 encoding)283 static u32 calc_l1_latency(u32 encoding)
284 {
285 if (encoding == 0x7)
286 return (65 * 1000); /* > 64us */
287 return (1000 << encoding);
288 }
289
290 /* Convert L1 acceptable latency encoding to ns */
calc_l1_acceptable(u32 encoding)291 static u32 calc_l1_acceptable(u32 encoding)
292 {
293 if (encoding == 0x7)
294 return -1U;
295 return (1000 << encoding);
296 }
297
298 struct aspm_register_info {
299 u32 support:2;
300 u32 enabled:2;
301 u32 latency_encoding_l0s;
302 u32 latency_encoding_l1;
303 };
304
pcie_get_aspm_reg(struct pci_dev * pdev,struct aspm_register_info * info)305 static void pcie_get_aspm_reg(struct pci_dev *pdev,
306 struct aspm_register_info *info)
307 {
308 int pos;
309 u16 reg16;
310 u32 reg32;
311
312 pos = pci_pcie_cap(pdev);
313 pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
314 info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
315 info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
316 info->latency_encoding_l1 = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
317 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
318 info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
319 }
320
pcie_aspm_check_latency(struct pci_dev * endpoint)321 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
322 {
323 u32 latency, l1_switch_latency = 0;
324 struct aspm_latency *acceptable;
325 struct pcie_link_state *link;
326
327 /* Device not in D0 doesn't need latency check */
328 if ((endpoint->current_state != PCI_D0) &&
329 (endpoint->current_state != PCI_UNKNOWN))
330 return;
331
332 link = endpoint->bus->self->link_state;
333 acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
334
335 while (link) {
336 /* Check upstream direction L0s latency */
337 if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
338 (link->latency_up.l0s > acceptable->l0s))
339 link->aspm_capable &= ~ASPM_STATE_L0S_UP;
340
341 /* Check downstream direction L0s latency */
342 if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
343 (link->latency_dw.l0s > acceptable->l0s))
344 link->aspm_capable &= ~ASPM_STATE_L0S_DW;
345 /*
346 * Check L1 latency.
347 * Every switch on the path to root complex need 1
348 * more microsecond for L1. Spec doesn't mention L0s.
349 */
350 latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
351 if ((link->aspm_capable & ASPM_STATE_L1) &&
352 (latency + l1_switch_latency > acceptable->l1))
353 link->aspm_capable &= ~ASPM_STATE_L1;
354 l1_switch_latency += 1000;
355
356 link = link->parent;
357 }
358 }
359
pcie_aspm_cap_init(struct pcie_link_state * link,int blacklist)360 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
361 {
362 struct pci_dev *child, *parent = link->pdev;
363 struct pci_bus *linkbus = parent->subordinate;
364 struct aspm_register_info upreg, dwreg;
365
366 if (blacklist) {
367 /* Set enabled/disable so that we will disable ASPM later */
368 link->aspm_enabled = ASPM_STATE_ALL;
369 link->aspm_disable = ASPM_STATE_ALL;
370 return;
371 }
372
373 /* Configure common clock before checking latencies */
374 pcie_aspm_configure_common_clock(link);
375
376 /* Get upstream/downstream components' register state */
377 pcie_get_aspm_reg(parent, &upreg);
378 child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
379 pcie_get_aspm_reg(child, &dwreg);
380
381 /*
382 * Setup L0s state
383 *
384 * Note that we must not enable L0s in either direction on a
385 * given link unless components on both sides of the link each
386 * support L0s.
387 */
388 if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
389 link->aspm_support |= ASPM_STATE_L0S;
390 if (dwreg.enabled & PCIE_LINK_STATE_L0S)
391 link->aspm_enabled |= ASPM_STATE_L0S_UP;
392 if (upreg.enabled & PCIE_LINK_STATE_L0S)
393 link->aspm_enabled |= ASPM_STATE_L0S_DW;
394 link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
395 link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
396
397 /* Setup L1 state */
398 if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
399 link->aspm_support |= ASPM_STATE_L1;
400 if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
401 link->aspm_enabled |= ASPM_STATE_L1;
402 link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
403 link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
404
405 /* Save default state */
406 link->aspm_default = link->aspm_enabled;
407
408 /* Setup initial capable state. Will be updated later */
409 link->aspm_capable = link->aspm_support;
410 /*
411 * If the downstream component has pci bridge function, don't
412 * do ASPM for now.
413 */
414 list_for_each_entry(child, &linkbus->devices, bus_list) {
415 if (child->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
416 link->aspm_disable = ASPM_STATE_ALL;
417 break;
418 }
419 }
420
421 /* Get and check endpoint acceptable latencies */
422 list_for_each_entry(child, &linkbus->devices, bus_list) {
423 int pos;
424 u32 reg32, encoding;
425 struct aspm_latency *acceptable =
426 &link->acceptable[PCI_FUNC(child->devfn)];
427
428 if (child->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
429 child->pcie_type != PCI_EXP_TYPE_LEG_END)
430 continue;
431
432 pos = pci_pcie_cap(child);
433 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
434 /* Calculate endpoint L0s acceptable latency */
435 encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
436 acceptable->l0s = calc_l0s_acceptable(encoding);
437 /* Calculate endpoint L1 acceptable latency */
438 encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
439 acceptable->l1 = calc_l1_acceptable(encoding);
440
441 pcie_aspm_check_latency(child);
442 }
443 }
444
pcie_config_aspm_dev(struct pci_dev * pdev,u32 val)445 static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
446 {
447 u16 reg16;
448 int pos = pci_pcie_cap(pdev);
449
450 pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
451 reg16 &= ~0x3;
452 reg16 |= val;
453 pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
454 }
455
pcie_config_aspm_link(struct pcie_link_state * link,u32 state)456 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
457 {
458 u32 upstream = 0, dwstream = 0;
459 struct pci_dev *child, *parent = link->pdev;
460 struct pci_bus *linkbus = parent->subordinate;
461
462 /* Nothing to do if the link is already in the requested state */
463 state &= (link->aspm_capable & ~link->aspm_disable);
464 if (link->aspm_enabled == state)
465 return;
466 /* Convert ASPM state to upstream/downstream ASPM register state */
467 if (state & ASPM_STATE_L0S_UP)
468 dwstream |= PCIE_LINK_STATE_L0S;
469 if (state & ASPM_STATE_L0S_DW)
470 upstream |= PCIE_LINK_STATE_L0S;
471 if (state & ASPM_STATE_L1) {
472 upstream |= PCIE_LINK_STATE_L1;
473 dwstream |= PCIE_LINK_STATE_L1;
474 }
475 /*
476 * Spec 2.0 suggests all functions should be configured the
477 * same setting for ASPM. Enabling ASPM L1 should be done in
478 * upstream component first and then downstream, and vice
479 * versa for disabling ASPM L1. Spec doesn't mention L0S.
480 */
481 if (state & ASPM_STATE_L1)
482 pcie_config_aspm_dev(parent, upstream);
483 list_for_each_entry(child, &linkbus->devices, bus_list)
484 pcie_config_aspm_dev(child, dwstream);
485 if (!(state & ASPM_STATE_L1))
486 pcie_config_aspm_dev(parent, upstream);
487
488 link->aspm_enabled = state;
489 }
490
pcie_config_aspm_path(struct pcie_link_state * link)491 static void pcie_config_aspm_path(struct pcie_link_state *link)
492 {
493 while (link) {
494 pcie_config_aspm_link(link, policy_to_aspm_state(link));
495 link = link->parent;
496 }
497 }
498
free_link_state(struct pcie_link_state * link)499 static void free_link_state(struct pcie_link_state *link)
500 {
501 link->pdev->link_state = NULL;
502 kfree(link);
503 }
504
pcie_aspm_sanity_check(struct pci_dev * pdev)505 static int pcie_aspm_sanity_check(struct pci_dev *pdev)
506 {
507 struct pci_dev *child;
508 int pos;
509 u32 reg32;
510
511 /*
512 * Some functions in a slot might not all be PCIe functions,
513 * very strange. Disable ASPM for the whole slot
514 */
515 list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
516 pos = pci_pcie_cap(child);
517 if (!pos)
518 return -EINVAL;
519
520 /*
521 * If ASPM is disabled then we're not going to change
522 * the BIOS state. It's safe to continue even if it's a
523 * pre-1.1 device
524 */
525
526 if (aspm_disabled)
527 continue;
528
529 /*
530 * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
531 * RBER bit to determine if a function is 1.1 version device
532 */
533 pci_read_config_dword(child, pos + PCI_EXP_DEVCAP, ®32);
534 if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
535 dev_printk(KERN_INFO, &child->dev, "disabling ASPM"
536 " on pre-1.1 PCIe device. You can enable it"
537 " with 'pcie_aspm=force'\n");
538 return -EINVAL;
539 }
540 }
541 return 0;
542 }
543
alloc_pcie_link_state(struct pci_dev * pdev)544 static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
545 {
546 struct pcie_link_state *link;
547
548 link = kzalloc(sizeof(*link), GFP_KERNEL);
549 if (!link)
550 return NULL;
551 INIT_LIST_HEAD(&link->sibling);
552 INIT_LIST_HEAD(&link->children);
553 INIT_LIST_HEAD(&link->link);
554 link->pdev = pdev;
555 if (pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM) {
556 struct pcie_link_state *parent;
557 parent = pdev->bus->parent->self->link_state;
558 if (!parent) {
559 kfree(link);
560 return NULL;
561 }
562 link->parent = parent;
563 list_add(&link->link, &parent->children);
564 }
565 /* Setup a pointer to the root port link */
566 if (!link->parent)
567 link->root = link;
568 else
569 link->root = link->parent->root;
570
571 list_add(&link->sibling, &link_list);
572 pdev->link_state = link;
573 return link;
574 }
575
576 /*
577 * pcie_aspm_init_link_state: Initiate PCI express link state.
578 * It is called after the pcie and its children devices are scaned.
579 * @pdev: the root port or switch downstream port
580 */
pcie_aspm_init_link_state(struct pci_dev * pdev)581 void pcie_aspm_init_link_state(struct pci_dev *pdev)
582 {
583 struct pcie_link_state *link;
584 int blacklist = !!pcie_aspm_sanity_check(pdev);
585
586 if (!aspm_support_enabled)
587 return;
588
589 if (!pci_is_pcie(pdev) || pdev->link_state)
590 return;
591 if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
592 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
593 return;
594
595 /* VIA has a strange chipset, root port is under a bridge */
596 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT &&
597 pdev->bus->self)
598 return;
599
600 down_read(&pci_bus_sem);
601 if (list_empty(&pdev->subordinate->devices))
602 goto out;
603
604 mutex_lock(&aspm_lock);
605 link = alloc_pcie_link_state(pdev);
606 if (!link)
607 goto unlock;
608 /*
609 * Setup initial ASPM state. Note that we need to configure
610 * upstream links also because capable state of them can be
611 * update through pcie_aspm_cap_init().
612 */
613 pcie_aspm_cap_init(link, blacklist);
614
615 /* Setup initial Clock PM state */
616 pcie_clkpm_cap_init(link, blacklist);
617
618 /*
619 * At this stage drivers haven't had an opportunity to change the
620 * link policy setting. Enabling ASPM on broken hardware can cripple
621 * it even before the driver has had a chance to disable ASPM, so
622 * default to a safe level right now. If we're enabling ASPM beyond
623 * the BIOS's expectation, we'll do so once pci_enable_device() is
624 * called.
625 */
626 if (aspm_policy != POLICY_POWERSAVE) {
627 pcie_config_aspm_path(link);
628 pcie_set_clkpm(link, policy_to_clkpm_state(link));
629 }
630
631 unlock:
632 mutex_unlock(&aspm_lock);
633 out:
634 up_read(&pci_bus_sem);
635 }
636
637 /* Recheck latencies and update aspm_capable for links under the root */
pcie_update_aspm_capable(struct pcie_link_state * root)638 static void pcie_update_aspm_capable(struct pcie_link_state *root)
639 {
640 struct pcie_link_state *link;
641 BUG_ON(root->parent);
642 list_for_each_entry(link, &link_list, sibling) {
643 if (link->root != root)
644 continue;
645 link->aspm_capable = link->aspm_support;
646 }
647 list_for_each_entry(link, &link_list, sibling) {
648 struct pci_dev *child;
649 struct pci_bus *linkbus = link->pdev->subordinate;
650 if (link->root != root)
651 continue;
652 list_for_each_entry(child, &linkbus->devices, bus_list) {
653 if ((child->pcie_type != PCI_EXP_TYPE_ENDPOINT) &&
654 (child->pcie_type != PCI_EXP_TYPE_LEG_END))
655 continue;
656 pcie_aspm_check_latency(child);
657 }
658 }
659 }
660
661 /* @pdev: the endpoint device */
pcie_aspm_exit_link_state(struct pci_dev * pdev)662 void pcie_aspm_exit_link_state(struct pci_dev *pdev)
663 {
664 struct pci_dev *parent = pdev->bus->self;
665 struct pcie_link_state *link, *root, *parent_link;
666
667 if (!pci_is_pcie(pdev) || !parent || !parent->link_state)
668 return;
669 if ((parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
670 (parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
671 return;
672
673 down_read(&pci_bus_sem);
674 mutex_lock(&aspm_lock);
675 /*
676 * All PCIe functions are in one slot, remove one function will remove
677 * the whole slot, so just wait until we are the last function left.
678 */
679 if (!list_is_last(&pdev->bus_list, &parent->subordinate->devices))
680 goto out;
681
682 link = parent->link_state;
683 root = link->root;
684 parent_link = link->parent;
685
686 /* All functions are removed, so just disable ASPM for the link */
687 pcie_config_aspm_link(link, 0);
688 list_del(&link->sibling);
689 list_del(&link->link);
690 /* Clock PM is for endpoint device */
691 free_link_state(link);
692
693 /* Recheck latencies and configure upstream links */
694 if (parent_link) {
695 pcie_update_aspm_capable(root);
696 pcie_config_aspm_path(parent_link);
697 }
698 out:
699 mutex_unlock(&aspm_lock);
700 up_read(&pci_bus_sem);
701 }
702
703 /* @pdev: the root port or switch downstream port */
pcie_aspm_pm_state_change(struct pci_dev * pdev)704 void pcie_aspm_pm_state_change(struct pci_dev *pdev)
705 {
706 struct pcie_link_state *link = pdev->link_state;
707
708 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
709 return;
710 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
711 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
712 return;
713 /*
714 * Devices changed PM state, we should recheck if latency
715 * meets all functions' requirement
716 */
717 down_read(&pci_bus_sem);
718 mutex_lock(&aspm_lock);
719 pcie_update_aspm_capable(link->root);
720 pcie_config_aspm_path(link);
721 mutex_unlock(&aspm_lock);
722 up_read(&pci_bus_sem);
723 }
724
pcie_aspm_powersave_config_link(struct pci_dev * pdev)725 void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
726 {
727 struct pcie_link_state *link = pdev->link_state;
728
729 if (aspm_disabled || !pci_is_pcie(pdev) || !link)
730 return;
731
732 if (aspm_policy != POLICY_POWERSAVE)
733 return;
734
735 if ((pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT) &&
736 (pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
737 return;
738
739 down_read(&pci_bus_sem);
740 mutex_lock(&aspm_lock);
741 pcie_config_aspm_path(link);
742 pcie_set_clkpm(link, policy_to_clkpm_state(link));
743 mutex_unlock(&aspm_lock);
744 up_read(&pci_bus_sem);
745 }
746
747 /*
748 * pci_disable_link_state - disable pci device's link state, so the link will
749 * never enter specific states
750 */
__pci_disable_link_state(struct pci_dev * pdev,int state,bool sem,bool force)751 static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem,
752 bool force)
753 {
754 struct pci_dev *parent = pdev->bus->self;
755 struct pcie_link_state *link;
756
757 if (aspm_disabled && !force)
758 return;
759
760 if (!pci_is_pcie(pdev))
761 return;
762
763 if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
764 pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
765 parent = pdev;
766 if (!parent || !parent->link_state)
767 return;
768
769 if (sem)
770 down_read(&pci_bus_sem);
771 mutex_lock(&aspm_lock);
772 link = parent->link_state;
773 if (state & PCIE_LINK_STATE_L0S)
774 link->aspm_disable |= ASPM_STATE_L0S;
775 if (state & PCIE_LINK_STATE_L1)
776 link->aspm_disable |= ASPM_STATE_L1;
777 pcie_config_aspm_link(link, policy_to_aspm_state(link));
778
779 if (state & PCIE_LINK_STATE_CLKPM) {
780 link->clkpm_capable = 0;
781 pcie_set_clkpm(link, 0);
782 }
783 mutex_unlock(&aspm_lock);
784 if (sem)
785 up_read(&pci_bus_sem);
786 }
787
pci_disable_link_state_locked(struct pci_dev * pdev,int state)788 void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
789 {
790 __pci_disable_link_state(pdev, state, false, false);
791 }
792 EXPORT_SYMBOL(pci_disable_link_state_locked);
793
pci_disable_link_state(struct pci_dev * pdev,int state)794 void pci_disable_link_state(struct pci_dev *pdev, int state)
795 {
796 __pci_disable_link_state(pdev, state, true, false);
797 }
798 EXPORT_SYMBOL(pci_disable_link_state);
799
pcie_clear_aspm(struct pci_bus * bus)800 void pcie_clear_aspm(struct pci_bus *bus)
801 {
802 struct pci_dev *child;
803
804 if (aspm_force)
805 return;
806
807 /*
808 * Clear any ASPM setup that the firmware has carried out on this bus
809 */
810 list_for_each_entry(child, &bus->devices, bus_list) {
811 __pci_disable_link_state(child, PCIE_LINK_STATE_L0S |
812 PCIE_LINK_STATE_L1 |
813 PCIE_LINK_STATE_CLKPM,
814 false, true);
815 }
816 }
817
pcie_aspm_set_policy(const char * val,struct kernel_param * kp)818 static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
819 {
820 int i;
821 struct pcie_link_state *link;
822
823 if (aspm_disabled)
824 return -EPERM;
825 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
826 if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
827 break;
828 if (i >= ARRAY_SIZE(policy_str))
829 return -EINVAL;
830 if (i == aspm_policy)
831 return 0;
832
833 down_read(&pci_bus_sem);
834 mutex_lock(&aspm_lock);
835 aspm_policy = i;
836 list_for_each_entry(link, &link_list, sibling) {
837 pcie_config_aspm_link(link, policy_to_aspm_state(link));
838 pcie_set_clkpm(link, policy_to_clkpm_state(link));
839 }
840 mutex_unlock(&aspm_lock);
841 up_read(&pci_bus_sem);
842 return 0;
843 }
844
pcie_aspm_get_policy(char * buffer,struct kernel_param * kp)845 static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
846 {
847 int i, cnt = 0;
848 for (i = 0; i < ARRAY_SIZE(policy_str); i++)
849 if (i == aspm_policy)
850 cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
851 else
852 cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
853 return cnt;
854 }
855
856 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
857 NULL, 0644);
858
859 #ifdef CONFIG_PCIEASPM_DEBUG
link_state_show(struct device * dev,struct device_attribute * attr,char * buf)860 static ssize_t link_state_show(struct device *dev,
861 struct device_attribute *attr,
862 char *buf)
863 {
864 struct pci_dev *pci_device = to_pci_dev(dev);
865 struct pcie_link_state *link_state = pci_device->link_state;
866
867 return sprintf(buf, "%d\n", link_state->aspm_enabled);
868 }
869
link_state_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t n)870 static ssize_t link_state_store(struct device *dev,
871 struct device_attribute *attr,
872 const char *buf,
873 size_t n)
874 {
875 struct pci_dev *pdev = to_pci_dev(dev);
876 struct pcie_link_state *link, *root = pdev->link_state->root;
877 u32 val = buf[0] - '0', state = 0;
878
879 if (aspm_disabled)
880 return -EPERM;
881 if (n < 1 || val > 3)
882 return -EINVAL;
883
884 /* Convert requested state to ASPM state */
885 if (val & PCIE_LINK_STATE_L0S)
886 state |= ASPM_STATE_L0S;
887 if (val & PCIE_LINK_STATE_L1)
888 state |= ASPM_STATE_L1;
889
890 down_read(&pci_bus_sem);
891 mutex_lock(&aspm_lock);
892 list_for_each_entry(link, &link_list, sibling) {
893 if (link->root != root)
894 continue;
895 pcie_config_aspm_link(link, state);
896 }
897 mutex_unlock(&aspm_lock);
898 up_read(&pci_bus_sem);
899 return n;
900 }
901
clk_ctl_show(struct device * dev,struct device_attribute * attr,char * buf)902 static ssize_t clk_ctl_show(struct device *dev,
903 struct device_attribute *attr,
904 char *buf)
905 {
906 struct pci_dev *pci_device = to_pci_dev(dev);
907 struct pcie_link_state *link_state = pci_device->link_state;
908
909 return sprintf(buf, "%d\n", link_state->clkpm_enabled);
910 }
911
clk_ctl_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t n)912 static ssize_t clk_ctl_store(struct device *dev,
913 struct device_attribute *attr,
914 const char *buf,
915 size_t n)
916 {
917 struct pci_dev *pdev = to_pci_dev(dev);
918 int state;
919
920 if (n < 1)
921 return -EINVAL;
922 state = buf[0]-'0';
923
924 down_read(&pci_bus_sem);
925 mutex_lock(&aspm_lock);
926 pcie_set_clkpm_nocheck(pdev->link_state, !!state);
927 mutex_unlock(&aspm_lock);
928 up_read(&pci_bus_sem);
929
930 return n;
931 }
932
933 static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
934 static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
935
936 static char power_group[] = "power";
pcie_aspm_create_sysfs_dev_files(struct pci_dev * pdev)937 void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
938 {
939 struct pcie_link_state *link_state = pdev->link_state;
940
941 if (!pci_is_pcie(pdev) ||
942 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
943 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
944 return;
945
946 if (link_state->aspm_support)
947 sysfs_add_file_to_group(&pdev->dev.kobj,
948 &dev_attr_link_state.attr, power_group);
949 if (link_state->clkpm_capable)
950 sysfs_add_file_to_group(&pdev->dev.kobj,
951 &dev_attr_clk_ctl.attr, power_group);
952 }
953
pcie_aspm_remove_sysfs_dev_files(struct pci_dev * pdev)954 void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
955 {
956 struct pcie_link_state *link_state = pdev->link_state;
957
958 if (!pci_is_pcie(pdev) ||
959 (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
960 pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
961 return;
962
963 if (link_state->aspm_support)
964 sysfs_remove_file_from_group(&pdev->dev.kobj,
965 &dev_attr_link_state.attr, power_group);
966 if (link_state->clkpm_capable)
967 sysfs_remove_file_from_group(&pdev->dev.kobj,
968 &dev_attr_clk_ctl.attr, power_group);
969 }
970 #endif
971
pcie_aspm_disable(char * str)972 static int __init pcie_aspm_disable(char *str)
973 {
974 if (!strcmp(str, "off")) {
975 aspm_policy = POLICY_DEFAULT;
976 aspm_disabled = 1;
977 aspm_support_enabled = false;
978 printk(KERN_INFO "PCIe ASPM is disabled\n");
979 } else if (!strcmp(str, "force")) {
980 aspm_force = 1;
981 printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
982 }
983 return 1;
984 }
985
986 __setup("pcie_aspm=", pcie_aspm_disable);
987
pcie_no_aspm(void)988 void pcie_no_aspm(void)
989 {
990 /*
991 * Disabling ASPM is intended to prevent the kernel from modifying
992 * existing hardware state, not to clear existing state. To that end:
993 * (a) set policy to POLICY_DEFAULT in order to avoid changing state
994 * (b) prevent userspace from changing policy
995 */
996 if (!aspm_force) {
997 aspm_policy = POLICY_DEFAULT;
998 aspm_disabled = 1;
999 }
1000 }
1001
1002 /**
1003 * pcie_aspm_enabled - is PCIe ASPM enabled?
1004 *
1005 * Returns true if ASPM has not been disabled by the command-line option
1006 * pcie_aspm=off.
1007 **/
pcie_aspm_enabled(void)1008 int pcie_aspm_enabled(void)
1009 {
1010 return !aspm_disabled;
1011 }
1012 EXPORT_SYMBOL(pcie_aspm_enabled);
1013
pcie_aspm_support_enabled(void)1014 bool pcie_aspm_support_enabled(void)
1015 {
1016 return aspm_support_enabled;
1017 }
1018 EXPORT_SYMBOL(pcie_aspm_support_enabled);
1019