1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Thunderbolt driver - Tunneling support
4 *
5 * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>
6 * Copyright (C) 2019, Intel Corporation
7 */
8
9 #include <linux/delay.h>
10 #include <linux/slab.h>
11 #include <linux/list.h>
12 #include <linux/ktime.h>
13 #include <linux/string_helpers.h>
14
15 #include "tunnel.h"
16 #include "tb.h"
17
18 /* PCIe adapters use always HopID of 8 for both directions */
19 #define TB_PCI_HOPID 8
20
21 #define TB_PCI_PATH_DOWN 0
22 #define TB_PCI_PATH_UP 1
23
24 /* USB3 adapters use always HopID of 8 for both directions */
25 #define TB_USB3_HOPID 8
26
27 #define TB_USB3_PATH_DOWN 0
28 #define TB_USB3_PATH_UP 1
29
30 /* DP adapters use HopID 8 for AUX and 9 for Video */
31 #define TB_DP_AUX_TX_HOPID 8
32 #define TB_DP_AUX_RX_HOPID 8
33 #define TB_DP_VIDEO_HOPID 9
34
35 #define TB_DP_VIDEO_PATH_OUT 0
36 #define TB_DP_AUX_PATH_OUT 1
37 #define TB_DP_AUX_PATH_IN 2
38
39 /* Minimum number of credits needed for PCIe path */
40 #define TB_MIN_PCIE_CREDITS 6U
41 /*
42 * Number of credits we try to allocate for each DMA path if not limited
43 * by the host router baMaxHI.
44 */
45 #define TB_DMA_CREDITS 14
46 /* Minimum number of credits for DMA path */
47 #define TB_MIN_DMA_CREDITS 1
48
49 static unsigned int dma_credits = TB_DMA_CREDITS;
50 module_param(dma_credits, uint, 0444);
51 MODULE_PARM_DESC(dma_credits, "specify custom credits for DMA tunnels (default: "
52 __MODULE_STRING(TB_DMA_CREDITS) ")");
53
54 static bool bw_alloc_mode = true;
55 module_param(bw_alloc_mode, bool, 0444);
56 MODULE_PARM_DESC(bw_alloc_mode,
57 "enable bandwidth allocation mode if supported (default: true)");
58
59 static const char * const tb_tunnel_names[] = { "PCI", "DP", "DMA", "USB3" };
60
61 #define __TB_TUNNEL_PRINT(level, tunnel, fmt, arg...) \
62 do { \
63 struct tb_tunnel *__tunnel = (tunnel); \
64 level(__tunnel->tb, "%llx:%u <-> %llx:%u (%s): " fmt, \
65 tb_route(__tunnel->src_port->sw), \
66 __tunnel->src_port->port, \
67 tb_route(__tunnel->dst_port->sw), \
68 __tunnel->dst_port->port, \
69 tb_tunnel_names[__tunnel->type], \
70 ## arg); \
71 } while (0)
72
73 #define tb_tunnel_WARN(tunnel, fmt, arg...) \
74 __TB_TUNNEL_PRINT(tb_WARN, tunnel, fmt, ##arg)
75 #define tb_tunnel_warn(tunnel, fmt, arg...) \
76 __TB_TUNNEL_PRINT(tb_warn, tunnel, fmt, ##arg)
77 #define tb_tunnel_info(tunnel, fmt, arg...) \
78 __TB_TUNNEL_PRINT(tb_info, tunnel, fmt, ##arg)
79 #define tb_tunnel_dbg(tunnel, fmt, arg...) \
80 __TB_TUNNEL_PRINT(tb_dbg, tunnel, fmt, ##arg)
81
tb_usable_credits(const struct tb_port * port)82 static inline unsigned int tb_usable_credits(const struct tb_port *port)
83 {
84 return port->total_credits - port->ctl_credits;
85 }
86
87 /**
88 * tb_available_credits() - Available credits for PCIe and DMA
89 * @port: Lane adapter to check
90 * @max_dp_streams: If non-%NULL stores maximum number of simultaneous DP
91 * streams possible through this lane adapter
92 */
tb_available_credits(const struct tb_port * port,size_t * max_dp_streams)93 static unsigned int tb_available_credits(const struct tb_port *port,
94 size_t *max_dp_streams)
95 {
96 const struct tb_switch *sw = port->sw;
97 int credits, usb3, pcie, spare;
98 size_t ndp;
99
100 usb3 = tb_acpi_may_tunnel_usb3() ? sw->max_usb3_credits : 0;
101 pcie = tb_acpi_may_tunnel_pcie() ? sw->max_pcie_credits : 0;
102
103 if (tb_acpi_is_xdomain_allowed()) {
104 spare = min_not_zero(sw->max_dma_credits, dma_credits);
105 /* Add some credits for potential second DMA tunnel */
106 spare += TB_MIN_DMA_CREDITS;
107 } else {
108 spare = 0;
109 }
110
111 credits = tb_usable_credits(port);
112 if (tb_acpi_may_tunnel_dp()) {
113 /*
114 * Maximum number of DP streams possible through the
115 * lane adapter.
116 */
117 if (sw->min_dp_aux_credits + sw->min_dp_main_credits)
118 ndp = (credits - (usb3 + pcie + spare)) /
119 (sw->min_dp_aux_credits + sw->min_dp_main_credits);
120 else
121 ndp = 0;
122 } else {
123 ndp = 0;
124 }
125 credits -= ndp * (sw->min_dp_aux_credits + sw->min_dp_main_credits);
126 credits -= usb3;
127
128 if (max_dp_streams)
129 *max_dp_streams = ndp;
130
131 return credits > 0 ? credits : 0;
132 }
133
tb_tunnel_alloc(struct tb * tb,size_t npaths,enum tb_tunnel_type type)134 static struct tb_tunnel *tb_tunnel_alloc(struct tb *tb, size_t npaths,
135 enum tb_tunnel_type type)
136 {
137 struct tb_tunnel *tunnel;
138
139 tunnel = kzalloc(sizeof(*tunnel), GFP_KERNEL);
140 if (!tunnel)
141 return NULL;
142
143 tunnel->paths = kcalloc(npaths, sizeof(tunnel->paths[0]), GFP_KERNEL);
144 if (!tunnel->paths) {
145 tb_tunnel_free(tunnel);
146 return NULL;
147 }
148
149 INIT_LIST_HEAD(&tunnel->list);
150 tunnel->tb = tb;
151 tunnel->npaths = npaths;
152 tunnel->type = type;
153
154 return tunnel;
155 }
156
tb_pci_set_ext_encapsulation(struct tb_tunnel * tunnel,bool enable)157 static int tb_pci_set_ext_encapsulation(struct tb_tunnel *tunnel, bool enable)
158 {
159 int ret;
160
161 /* Only supported of both routers are at least USB4 v2 */
162 if (usb4_switch_version(tunnel->src_port->sw) < 2 ||
163 usb4_switch_version(tunnel->dst_port->sw) < 2)
164 return 0;
165
166 ret = usb4_pci_port_set_ext_encapsulation(tunnel->src_port, enable);
167 if (ret)
168 return ret;
169
170 ret = usb4_pci_port_set_ext_encapsulation(tunnel->dst_port, enable);
171 if (ret)
172 return ret;
173
174 tb_tunnel_dbg(tunnel, "extended encapsulation %s\n",
175 str_enabled_disabled(enable));
176 return 0;
177 }
178
tb_pci_activate(struct tb_tunnel * tunnel,bool activate)179 static int tb_pci_activate(struct tb_tunnel *tunnel, bool activate)
180 {
181 int res;
182
183 if (activate) {
184 res = tb_pci_set_ext_encapsulation(tunnel, activate);
185 if (res)
186 return res;
187 }
188
189 res = tb_pci_port_enable(tunnel->src_port, activate);
190 if (res)
191 return res;
192
193 if (tb_port_is_pcie_up(tunnel->dst_port)) {
194 res = tb_pci_port_enable(tunnel->dst_port, activate);
195 if (res)
196 return res;
197 }
198
199 return activate ? 0 : tb_pci_set_ext_encapsulation(tunnel, activate);
200 }
201
tb_pci_init_credits(struct tb_path_hop * hop)202 static int tb_pci_init_credits(struct tb_path_hop *hop)
203 {
204 struct tb_port *port = hop->in_port;
205 struct tb_switch *sw = port->sw;
206 unsigned int credits;
207
208 if (tb_port_use_credit_allocation(port)) {
209 unsigned int available;
210
211 available = tb_available_credits(port, NULL);
212 credits = min(sw->max_pcie_credits, available);
213
214 if (credits < TB_MIN_PCIE_CREDITS)
215 return -ENOSPC;
216
217 credits = max(TB_MIN_PCIE_CREDITS, credits);
218 } else {
219 if (tb_port_is_null(port))
220 credits = port->bonded ? 32 : 16;
221 else
222 credits = 7;
223 }
224
225 hop->initial_credits = credits;
226 return 0;
227 }
228
tb_pci_init_path(struct tb_path * path)229 static int tb_pci_init_path(struct tb_path *path)
230 {
231 struct tb_path_hop *hop;
232
233 path->egress_fc_enable = TB_PATH_SOURCE | TB_PATH_INTERNAL;
234 path->egress_shared_buffer = TB_PATH_NONE;
235 path->ingress_fc_enable = TB_PATH_ALL;
236 path->ingress_shared_buffer = TB_PATH_NONE;
237 path->priority = 3;
238 path->weight = 1;
239 path->drop_packages = 0;
240
241 tb_path_for_each_hop(path, hop) {
242 int ret;
243
244 ret = tb_pci_init_credits(hop);
245 if (ret)
246 return ret;
247 }
248
249 return 0;
250 }
251
252 /**
253 * tb_tunnel_discover_pci() - Discover existing PCIe tunnels
254 * @tb: Pointer to the domain structure
255 * @down: PCIe downstream adapter
256 * @alloc_hopid: Allocate HopIDs from visited ports
257 *
258 * If @down adapter is active, follows the tunnel to the PCIe upstream
259 * adapter and back. Returns the discovered tunnel or %NULL if there was
260 * no tunnel.
261 */
tb_tunnel_discover_pci(struct tb * tb,struct tb_port * down,bool alloc_hopid)262 struct tb_tunnel *tb_tunnel_discover_pci(struct tb *tb, struct tb_port *down,
263 bool alloc_hopid)
264 {
265 struct tb_tunnel *tunnel;
266 struct tb_path *path;
267
268 if (!tb_pci_port_is_enabled(down))
269 return NULL;
270
271 tunnel = tb_tunnel_alloc(tb, 2, TB_TUNNEL_PCI);
272 if (!tunnel)
273 return NULL;
274
275 tunnel->activate = tb_pci_activate;
276 tunnel->src_port = down;
277
278 /*
279 * Discover both paths even if they are not complete. We will
280 * clean them up by calling tb_tunnel_deactivate() below in that
281 * case.
282 */
283 path = tb_path_discover(down, TB_PCI_HOPID, NULL, -1,
284 &tunnel->dst_port, "PCIe Up", alloc_hopid);
285 if (!path) {
286 /* Just disable the downstream port */
287 tb_pci_port_enable(down, false);
288 goto err_free;
289 }
290 tunnel->paths[TB_PCI_PATH_UP] = path;
291 if (tb_pci_init_path(tunnel->paths[TB_PCI_PATH_UP]))
292 goto err_free;
293
294 path = tb_path_discover(tunnel->dst_port, -1, down, TB_PCI_HOPID, NULL,
295 "PCIe Down", alloc_hopid);
296 if (!path)
297 goto err_deactivate;
298 tunnel->paths[TB_PCI_PATH_DOWN] = path;
299 if (tb_pci_init_path(tunnel->paths[TB_PCI_PATH_DOWN]))
300 goto err_deactivate;
301
302 /* Validate that the tunnel is complete */
303 if (!tb_port_is_pcie_up(tunnel->dst_port)) {
304 tb_port_warn(tunnel->dst_port,
305 "path does not end on a PCIe adapter, cleaning up\n");
306 goto err_deactivate;
307 }
308
309 if (down != tunnel->src_port) {
310 tb_tunnel_warn(tunnel, "path is not complete, cleaning up\n");
311 goto err_deactivate;
312 }
313
314 if (!tb_pci_port_is_enabled(tunnel->dst_port)) {
315 tb_tunnel_warn(tunnel,
316 "tunnel is not fully activated, cleaning up\n");
317 goto err_deactivate;
318 }
319
320 tb_tunnel_dbg(tunnel, "discovered\n");
321 return tunnel;
322
323 err_deactivate:
324 tb_tunnel_deactivate(tunnel);
325 err_free:
326 tb_tunnel_free(tunnel);
327
328 return NULL;
329 }
330
331 /**
332 * tb_tunnel_alloc_pci() - allocate a pci tunnel
333 * @tb: Pointer to the domain structure
334 * @up: PCIe upstream adapter port
335 * @down: PCIe downstream adapter port
336 *
337 * Allocate a PCI tunnel. The ports must be of type TB_TYPE_PCIE_UP and
338 * TB_TYPE_PCIE_DOWN.
339 *
340 * Return: Returns a tb_tunnel on success or NULL on failure.
341 */
tb_tunnel_alloc_pci(struct tb * tb,struct tb_port * up,struct tb_port * down)342 struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up,
343 struct tb_port *down)
344 {
345 struct tb_tunnel *tunnel;
346 struct tb_path *path;
347
348 tunnel = tb_tunnel_alloc(tb, 2, TB_TUNNEL_PCI);
349 if (!tunnel)
350 return NULL;
351
352 tunnel->activate = tb_pci_activate;
353 tunnel->src_port = down;
354 tunnel->dst_port = up;
355
356 path = tb_path_alloc(tb, down, TB_PCI_HOPID, up, TB_PCI_HOPID, 0,
357 "PCIe Down");
358 if (!path)
359 goto err_free;
360 tunnel->paths[TB_PCI_PATH_DOWN] = path;
361 if (tb_pci_init_path(path))
362 goto err_free;
363
364 path = tb_path_alloc(tb, up, TB_PCI_HOPID, down, TB_PCI_HOPID, 0,
365 "PCIe Up");
366 if (!path)
367 goto err_free;
368 tunnel->paths[TB_PCI_PATH_UP] = path;
369 if (tb_pci_init_path(path))
370 goto err_free;
371
372 return tunnel;
373
374 err_free:
375 tb_tunnel_free(tunnel);
376 return NULL;
377 }
378
tb_dp_is_usb4(const struct tb_switch * sw)379 static bool tb_dp_is_usb4(const struct tb_switch *sw)
380 {
381 /* Titan Ridge DP adapters need the same treatment as USB4 */
382 return tb_switch_is_usb4(sw) || tb_switch_is_titan_ridge(sw);
383 }
384
tb_dp_cm_handshake(struct tb_port * in,struct tb_port * out,int timeout_msec)385 static int tb_dp_cm_handshake(struct tb_port *in, struct tb_port *out,
386 int timeout_msec)
387 {
388 ktime_t timeout = ktime_add_ms(ktime_get(), timeout_msec);
389 u32 val;
390 int ret;
391
392 /* Both ends need to support this */
393 if (!tb_dp_is_usb4(in->sw) || !tb_dp_is_usb4(out->sw))
394 return 0;
395
396 ret = tb_port_read(out, &val, TB_CFG_PORT,
397 out->cap_adap + DP_STATUS_CTRL, 1);
398 if (ret)
399 return ret;
400
401 val |= DP_STATUS_CTRL_UF | DP_STATUS_CTRL_CMHS;
402
403 ret = tb_port_write(out, &val, TB_CFG_PORT,
404 out->cap_adap + DP_STATUS_CTRL, 1);
405 if (ret)
406 return ret;
407
408 do {
409 ret = tb_port_read(out, &val, TB_CFG_PORT,
410 out->cap_adap + DP_STATUS_CTRL, 1);
411 if (ret)
412 return ret;
413 if (!(val & DP_STATUS_CTRL_CMHS))
414 return 0;
415 usleep_range(100, 150);
416 } while (ktime_before(ktime_get(), timeout));
417
418 return -ETIMEDOUT;
419 }
420
421 /*
422 * Returns maximum possible rate from capability supporting only DP 2.0
423 * and below. Used when DP BW allocation mode is not enabled.
424 */
tb_dp_cap_get_rate(u32 val)425 static inline u32 tb_dp_cap_get_rate(u32 val)
426 {
427 u32 rate = (val & DP_COMMON_CAP_RATE_MASK) >> DP_COMMON_CAP_RATE_SHIFT;
428
429 switch (rate) {
430 case DP_COMMON_CAP_RATE_RBR:
431 return 1620;
432 case DP_COMMON_CAP_RATE_HBR:
433 return 2700;
434 case DP_COMMON_CAP_RATE_HBR2:
435 return 5400;
436 case DP_COMMON_CAP_RATE_HBR3:
437 return 8100;
438 default:
439 return 0;
440 }
441 }
442
443 /*
444 * Returns maximum possible rate from capability supporting DP 2.1
445 * UHBR20, 13.5 and 10 rates as well. Use only when DP BW allocation
446 * mode is enabled.
447 */
tb_dp_cap_get_rate_ext(u32 val)448 static inline u32 tb_dp_cap_get_rate_ext(u32 val)
449 {
450 if (val & DP_COMMON_CAP_UHBR20)
451 return 20000;
452 else if (val & DP_COMMON_CAP_UHBR13_5)
453 return 13500;
454 else if (val & DP_COMMON_CAP_UHBR10)
455 return 10000;
456
457 return tb_dp_cap_get_rate(val);
458 }
459
tb_dp_is_uhbr_rate(unsigned int rate)460 static inline bool tb_dp_is_uhbr_rate(unsigned int rate)
461 {
462 return rate >= 10000;
463 }
464
tb_dp_cap_set_rate(u32 val,u32 rate)465 static inline u32 tb_dp_cap_set_rate(u32 val, u32 rate)
466 {
467 val &= ~DP_COMMON_CAP_RATE_MASK;
468 switch (rate) {
469 default:
470 WARN(1, "invalid rate %u passed, defaulting to 1620 MB/s\n", rate);
471 fallthrough;
472 case 1620:
473 val |= DP_COMMON_CAP_RATE_RBR << DP_COMMON_CAP_RATE_SHIFT;
474 break;
475 case 2700:
476 val |= DP_COMMON_CAP_RATE_HBR << DP_COMMON_CAP_RATE_SHIFT;
477 break;
478 case 5400:
479 val |= DP_COMMON_CAP_RATE_HBR2 << DP_COMMON_CAP_RATE_SHIFT;
480 break;
481 case 8100:
482 val |= DP_COMMON_CAP_RATE_HBR3 << DP_COMMON_CAP_RATE_SHIFT;
483 break;
484 }
485 return val;
486 }
487
tb_dp_cap_get_lanes(u32 val)488 static inline u32 tb_dp_cap_get_lanes(u32 val)
489 {
490 u32 lanes = (val & DP_COMMON_CAP_LANES_MASK) >> DP_COMMON_CAP_LANES_SHIFT;
491
492 switch (lanes) {
493 case DP_COMMON_CAP_1_LANE:
494 return 1;
495 case DP_COMMON_CAP_2_LANES:
496 return 2;
497 case DP_COMMON_CAP_4_LANES:
498 return 4;
499 default:
500 return 0;
501 }
502 }
503
tb_dp_cap_set_lanes(u32 val,u32 lanes)504 static inline u32 tb_dp_cap_set_lanes(u32 val, u32 lanes)
505 {
506 val &= ~DP_COMMON_CAP_LANES_MASK;
507 switch (lanes) {
508 default:
509 WARN(1, "invalid number of lanes %u passed, defaulting to 1\n",
510 lanes);
511 fallthrough;
512 case 1:
513 val |= DP_COMMON_CAP_1_LANE << DP_COMMON_CAP_LANES_SHIFT;
514 break;
515 case 2:
516 val |= DP_COMMON_CAP_2_LANES << DP_COMMON_CAP_LANES_SHIFT;
517 break;
518 case 4:
519 val |= DP_COMMON_CAP_4_LANES << DP_COMMON_CAP_LANES_SHIFT;
520 break;
521 }
522 return val;
523 }
524
tb_dp_bandwidth(unsigned int rate,unsigned int lanes)525 static unsigned int tb_dp_bandwidth(unsigned int rate, unsigned int lanes)
526 {
527 /* Tunneling removes the DP 8b/10b 128/132b encoding */
528 if (tb_dp_is_uhbr_rate(rate))
529 return rate * lanes * 128 / 132;
530 return rate * lanes * 8 / 10;
531 }
532
tb_dp_reduce_bandwidth(int max_bw,u32 in_rate,u32 in_lanes,u32 out_rate,u32 out_lanes,u32 * new_rate,u32 * new_lanes)533 static int tb_dp_reduce_bandwidth(int max_bw, u32 in_rate, u32 in_lanes,
534 u32 out_rate, u32 out_lanes, u32 *new_rate,
535 u32 *new_lanes)
536 {
537 static const u32 dp_bw[][2] = {
538 /* Mb/s, lanes */
539 { 8100, 4 }, /* 25920 Mb/s */
540 { 5400, 4 }, /* 17280 Mb/s */
541 { 8100, 2 }, /* 12960 Mb/s */
542 { 2700, 4 }, /* 8640 Mb/s */
543 { 5400, 2 }, /* 8640 Mb/s */
544 { 8100, 1 }, /* 6480 Mb/s */
545 { 1620, 4 }, /* 5184 Mb/s */
546 { 5400, 1 }, /* 4320 Mb/s */
547 { 2700, 2 }, /* 4320 Mb/s */
548 { 1620, 2 }, /* 2592 Mb/s */
549 { 2700, 1 }, /* 2160 Mb/s */
550 { 1620, 1 }, /* 1296 Mb/s */
551 };
552 unsigned int i;
553
554 /*
555 * Find a combination that can fit into max_bw and does not
556 * exceed the maximum rate and lanes supported by the DP OUT and
557 * DP IN adapters.
558 */
559 for (i = 0; i < ARRAY_SIZE(dp_bw); i++) {
560 if (dp_bw[i][0] > out_rate || dp_bw[i][1] > out_lanes)
561 continue;
562
563 if (dp_bw[i][0] > in_rate || dp_bw[i][1] > in_lanes)
564 continue;
565
566 if (tb_dp_bandwidth(dp_bw[i][0], dp_bw[i][1]) <= max_bw) {
567 *new_rate = dp_bw[i][0];
568 *new_lanes = dp_bw[i][1];
569 return 0;
570 }
571 }
572
573 return -ENOSR;
574 }
575
tb_dp_xchg_caps(struct tb_tunnel * tunnel)576 static int tb_dp_xchg_caps(struct tb_tunnel *tunnel)
577 {
578 u32 out_dp_cap, out_rate, out_lanes, in_dp_cap, in_rate, in_lanes, bw;
579 struct tb_port *out = tunnel->dst_port;
580 struct tb_port *in = tunnel->src_port;
581 int ret, max_bw;
582
583 /*
584 * Copy DP_LOCAL_CAP register to DP_REMOTE_CAP register for
585 * newer generation hardware.
586 */
587 if (in->sw->generation < 2 || out->sw->generation < 2)
588 return 0;
589
590 /*
591 * Perform connection manager handshake between IN and OUT ports
592 * before capabilities exchange can take place.
593 */
594 ret = tb_dp_cm_handshake(in, out, 3000);
595 if (ret)
596 return ret;
597
598 /* Read both DP_LOCAL_CAP registers */
599 ret = tb_port_read(in, &in_dp_cap, TB_CFG_PORT,
600 in->cap_adap + DP_LOCAL_CAP, 1);
601 if (ret)
602 return ret;
603
604 ret = tb_port_read(out, &out_dp_cap, TB_CFG_PORT,
605 out->cap_adap + DP_LOCAL_CAP, 1);
606 if (ret)
607 return ret;
608
609 /* Write IN local caps to OUT remote caps */
610 ret = tb_port_write(out, &in_dp_cap, TB_CFG_PORT,
611 out->cap_adap + DP_REMOTE_CAP, 1);
612 if (ret)
613 return ret;
614
615 in_rate = tb_dp_cap_get_rate(in_dp_cap);
616 in_lanes = tb_dp_cap_get_lanes(in_dp_cap);
617 tb_port_dbg(in, "maximum supported bandwidth %u Mb/s x%u = %u Mb/s\n",
618 in_rate, in_lanes, tb_dp_bandwidth(in_rate, in_lanes));
619
620 /*
621 * If the tunnel bandwidth is limited (max_bw is set) then see
622 * if we need to reduce bandwidth to fit there.
623 */
624 out_rate = tb_dp_cap_get_rate(out_dp_cap);
625 out_lanes = tb_dp_cap_get_lanes(out_dp_cap);
626 bw = tb_dp_bandwidth(out_rate, out_lanes);
627 tb_port_dbg(out, "maximum supported bandwidth %u Mb/s x%u = %u Mb/s\n",
628 out_rate, out_lanes, bw);
629
630 if (in->sw->config.depth < out->sw->config.depth)
631 max_bw = tunnel->max_down;
632 else
633 max_bw = tunnel->max_up;
634
635 if (max_bw && bw > max_bw) {
636 u32 new_rate, new_lanes, new_bw;
637
638 ret = tb_dp_reduce_bandwidth(max_bw, in_rate, in_lanes,
639 out_rate, out_lanes, &new_rate,
640 &new_lanes);
641 if (ret) {
642 tb_port_info(out, "not enough bandwidth for DP tunnel\n");
643 return ret;
644 }
645
646 new_bw = tb_dp_bandwidth(new_rate, new_lanes);
647 tb_port_dbg(out, "bandwidth reduced to %u Mb/s x%u = %u Mb/s\n",
648 new_rate, new_lanes, new_bw);
649
650 /*
651 * Set new rate and number of lanes before writing it to
652 * the IN port remote caps.
653 */
654 out_dp_cap = tb_dp_cap_set_rate(out_dp_cap, new_rate);
655 out_dp_cap = tb_dp_cap_set_lanes(out_dp_cap, new_lanes);
656 }
657
658 /*
659 * Titan Ridge does not disable AUX timers when it gets
660 * SET_CONFIG with SET_LTTPR_MODE set. This causes problems with
661 * DP tunneling.
662 */
663 if (tb_route(out->sw) && tb_switch_is_titan_ridge(out->sw)) {
664 out_dp_cap |= DP_COMMON_CAP_LTTPR_NS;
665 tb_port_dbg(out, "disabling LTTPR\n");
666 }
667
668 return tb_port_write(in, &out_dp_cap, TB_CFG_PORT,
669 in->cap_adap + DP_REMOTE_CAP, 1);
670 }
671
tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel * tunnel)672 static int tb_dp_bandwidth_alloc_mode_enable(struct tb_tunnel *tunnel)
673 {
674 int ret, estimated_bw, granularity, tmp;
675 struct tb_port *out = tunnel->dst_port;
676 struct tb_port *in = tunnel->src_port;
677 u32 out_dp_cap, out_rate, out_lanes;
678 u32 in_dp_cap, in_rate, in_lanes;
679 u32 rate, lanes;
680
681 if (!bw_alloc_mode)
682 return 0;
683
684 ret = usb4_dp_port_set_cm_bandwidth_mode_supported(in, true);
685 if (ret)
686 return ret;
687
688 ret = usb4_dp_port_set_group_id(in, in->group->index);
689 if (ret)
690 return ret;
691
692 /*
693 * Get the non-reduced rate and lanes based on the lowest
694 * capability of both adapters.
695 */
696 ret = tb_port_read(in, &in_dp_cap, TB_CFG_PORT,
697 in->cap_adap + DP_LOCAL_CAP, 1);
698 if (ret)
699 return ret;
700
701 ret = tb_port_read(out, &out_dp_cap, TB_CFG_PORT,
702 out->cap_adap + DP_LOCAL_CAP, 1);
703 if (ret)
704 return ret;
705
706 in_rate = tb_dp_cap_get_rate(in_dp_cap);
707 in_lanes = tb_dp_cap_get_lanes(in_dp_cap);
708 out_rate = tb_dp_cap_get_rate(out_dp_cap);
709 out_lanes = tb_dp_cap_get_lanes(out_dp_cap);
710
711 rate = min(in_rate, out_rate);
712 lanes = min(in_lanes, out_lanes);
713 tmp = tb_dp_bandwidth(rate, lanes);
714
715 tb_port_dbg(in, "non-reduced bandwidth %u Mb/s x%u = %u Mb/s\n", rate,
716 lanes, tmp);
717
718 ret = usb4_dp_port_set_nrd(in, rate, lanes);
719 if (ret)
720 return ret;
721
722 /*
723 * Pick up granularity that supports maximum possible bandwidth.
724 * For that we use the UHBR rates too.
725 */
726 in_rate = tb_dp_cap_get_rate_ext(in_dp_cap);
727 out_rate = tb_dp_cap_get_rate_ext(out_dp_cap);
728 rate = min(in_rate, out_rate);
729 tmp = tb_dp_bandwidth(rate, lanes);
730
731 tb_port_dbg(in,
732 "maximum bandwidth through allocation mode %u Mb/s x%u = %u Mb/s\n",
733 rate, lanes, tmp);
734
735 for (granularity = 250; tmp / granularity > 255 && granularity <= 1000;
736 granularity *= 2)
737 ;
738
739 tb_port_dbg(in, "granularity %d Mb/s\n", granularity);
740
741 /*
742 * Returns -EINVAL if granularity above is outside of the
743 * accepted ranges.
744 */
745 ret = usb4_dp_port_set_granularity(in, granularity);
746 if (ret)
747 return ret;
748
749 /*
750 * Bandwidth estimation is pretty much what we have in
751 * max_up/down fields. For discovery we just read what the
752 * estimation was set to.
753 */
754 if (in->sw->config.depth < out->sw->config.depth)
755 estimated_bw = tunnel->max_down;
756 else
757 estimated_bw = tunnel->max_up;
758
759 tb_port_dbg(in, "estimated bandwidth %d Mb/s\n", estimated_bw);
760
761 ret = usb4_dp_port_set_estimated_bandwidth(in, estimated_bw);
762 if (ret)
763 return ret;
764
765 /* Initial allocation should be 0 according the spec */
766 ret = usb4_dp_port_allocate_bandwidth(in, 0);
767 if (ret)
768 return ret;
769
770 tb_port_dbg(in, "bandwidth allocation mode enabled\n");
771 return 0;
772 }
773
tb_dp_init(struct tb_tunnel * tunnel)774 static int tb_dp_init(struct tb_tunnel *tunnel)
775 {
776 struct tb_port *in = tunnel->src_port;
777 struct tb_switch *sw = in->sw;
778 struct tb *tb = in->sw->tb;
779 int ret;
780
781 ret = tb_dp_xchg_caps(tunnel);
782 if (ret)
783 return ret;
784
785 if (!tb_switch_is_usb4(sw))
786 return 0;
787
788 if (!usb4_dp_port_bandwidth_mode_supported(in))
789 return 0;
790
791 tb_port_dbg(in, "bandwidth allocation mode supported\n");
792
793 ret = usb4_dp_port_set_cm_id(in, tb->index);
794 if (ret)
795 return ret;
796
797 return tb_dp_bandwidth_alloc_mode_enable(tunnel);
798 }
799
tb_dp_deinit(struct tb_tunnel * tunnel)800 static void tb_dp_deinit(struct tb_tunnel *tunnel)
801 {
802 struct tb_port *in = tunnel->src_port;
803
804 if (!usb4_dp_port_bandwidth_mode_supported(in))
805 return;
806 if (usb4_dp_port_bandwidth_mode_enabled(in)) {
807 usb4_dp_port_set_cm_bandwidth_mode_supported(in, false);
808 tb_port_dbg(in, "bandwidth allocation mode disabled\n");
809 }
810 }
811
tb_dp_activate(struct tb_tunnel * tunnel,bool active)812 static int tb_dp_activate(struct tb_tunnel *tunnel, bool active)
813 {
814 int ret;
815
816 if (active) {
817 struct tb_path **paths;
818 int last;
819
820 paths = tunnel->paths;
821 last = paths[TB_DP_VIDEO_PATH_OUT]->path_length - 1;
822
823 tb_dp_port_set_hops(tunnel->src_port,
824 paths[TB_DP_VIDEO_PATH_OUT]->hops[0].in_hop_index,
825 paths[TB_DP_AUX_PATH_OUT]->hops[0].in_hop_index,
826 paths[TB_DP_AUX_PATH_IN]->hops[last].next_hop_index);
827
828 tb_dp_port_set_hops(tunnel->dst_port,
829 paths[TB_DP_VIDEO_PATH_OUT]->hops[last].next_hop_index,
830 paths[TB_DP_AUX_PATH_IN]->hops[0].in_hop_index,
831 paths[TB_DP_AUX_PATH_OUT]->hops[last].next_hop_index);
832 } else {
833 tb_dp_port_hpd_clear(tunnel->src_port);
834 tb_dp_port_set_hops(tunnel->src_port, 0, 0, 0);
835 if (tb_port_is_dpout(tunnel->dst_port))
836 tb_dp_port_set_hops(tunnel->dst_port, 0, 0, 0);
837 }
838
839 ret = tb_dp_port_enable(tunnel->src_port, active);
840 if (ret)
841 return ret;
842
843 if (tb_port_is_dpout(tunnel->dst_port))
844 return tb_dp_port_enable(tunnel->dst_port, active);
845
846 return 0;
847 }
848
849 /* max_bw is rounded up to next granularity */
tb_dp_bandwidth_mode_maximum_bandwidth(struct tb_tunnel * tunnel,int * max_bw)850 static int tb_dp_bandwidth_mode_maximum_bandwidth(struct tb_tunnel *tunnel,
851 int *max_bw)
852 {
853 struct tb_port *in = tunnel->src_port;
854 int ret, rate, lanes, nrd_bw;
855 u32 cap;
856
857 /*
858 * DP IN adapter DP_LOCAL_CAP gets updated to the lowest AUX
859 * read parameter values so this so we can use this to determine
860 * the maximum possible bandwidth over this link.
861 *
862 * See USB4 v2 spec 1.0 10.4.4.5.
863 */
864 ret = tb_port_read(in, &cap, TB_CFG_PORT,
865 in->cap_adap + DP_LOCAL_CAP, 1);
866 if (ret)
867 return ret;
868
869 rate = tb_dp_cap_get_rate_ext(cap);
870 if (tb_dp_is_uhbr_rate(rate)) {
871 /*
872 * When UHBR is used there is no reduction in lanes so
873 * we can use this directly.
874 */
875 lanes = tb_dp_cap_get_lanes(cap);
876 } else {
877 /*
878 * If there is no UHBR supported then check the
879 * non-reduced rate and lanes.
880 */
881 ret = usb4_dp_port_nrd(in, &rate, &lanes);
882 if (ret)
883 return ret;
884 }
885
886 nrd_bw = tb_dp_bandwidth(rate, lanes);
887
888 if (max_bw) {
889 ret = usb4_dp_port_granularity(in);
890 if (ret < 0)
891 return ret;
892 *max_bw = roundup(nrd_bw, ret);
893 }
894
895 return nrd_bw;
896 }
897
tb_dp_bandwidth_mode_consumed_bandwidth(struct tb_tunnel * tunnel,int * consumed_up,int * consumed_down)898 static int tb_dp_bandwidth_mode_consumed_bandwidth(struct tb_tunnel *tunnel,
899 int *consumed_up,
900 int *consumed_down)
901 {
902 struct tb_port *out = tunnel->dst_port;
903 struct tb_port *in = tunnel->src_port;
904 int ret, allocated_bw, max_bw;
905
906 if (!usb4_dp_port_bandwidth_mode_enabled(in))
907 return -EOPNOTSUPP;
908
909 if (!tunnel->bw_mode)
910 return -EOPNOTSUPP;
911
912 /* Read what was allocated previously if any */
913 ret = usb4_dp_port_allocated_bandwidth(in);
914 if (ret < 0)
915 return ret;
916 allocated_bw = ret;
917
918 ret = tb_dp_bandwidth_mode_maximum_bandwidth(tunnel, &max_bw);
919 if (ret < 0)
920 return ret;
921 if (allocated_bw == max_bw)
922 allocated_bw = ret;
923
924 tb_port_dbg(in, "consumed bandwidth through allocation mode %d Mb/s\n",
925 allocated_bw);
926
927 if (in->sw->config.depth < out->sw->config.depth) {
928 *consumed_up = 0;
929 *consumed_down = allocated_bw;
930 } else {
931 *consumed_up = allocated_bw;
932 *consumed_down = 0;
933 }
934
935 return 0;
936 }
937
tb_dp_allocated_bandwidth(struct tb_tunnel * tunnel,int * allocated_up,int * allocated_down)938 static int tb_dp_allocated_bandwidth(struct tb_tunnel *tunnel, int *allocated_up,
939 int *allocated_down)
940 {
941 struct tb_port *out = tunnel->dst_port;
942 struct tb_port *in = tunnel->src_port;
943
944 /*
945 * If we have already set the allocated bandwidth then use that.
946 * Otherwise we read it from the DPRX.
947 */
948 if (usb4_dp_port_bandwidth_mode_enabled(in) && tunnel->bw_mode) {
949 int ret, allocated_bw, max_bw;
950
951 ret = usb4_dp_port_allocated_bandwidth(in);
952 if (ret < 0)
953 return ret;
954 allocated_bw = ret;
955
956 ret = tb_dp_bandwidth_mode_maximum_bandwidth(tunnel, &max_bw);
957 if (ret < 0)
958 return ret;
959 if (allocated_bw == max_bw)
960 allocated_bw = ret;
961
962 if (in->sw->config.depth < out->sw->config.depth) {
963 *allocated_up = 0;
964 *allocated_down = allocated_bw;
965 } else {
966 *allocated_up = allocated_bw;
967 *allocated_down = 0;
968 }
969 return 0;
970 }
971
972 return tunnel->consumed_bandwidth(tunnel, allocated_up,
973 allocated_down);
974 }
975
tb_dp_alloc_bandwidth(struct tb_tunnel * tunnel,int * alloc_up,int * alloc_down)976 static int tb_dp_alloc_bandwidth(struct tb_tunnel *tunnel, int *alloc_up,
977 int *alloc_down)
978 {
979 struct tb_port *out = tunnel->dst_port;
980 struct tb_port *in = tunnel->src_port;
981 int max_bw, ret, tmp;
982
983 if (!usb4_dp_port_bandwidth_mode_enabled(in))
984 return -EOPNOTSUPP;
985
986 ret = tb_dp_bandwidth_mode_maximum_bandwidth(tunnel, &max_bw);
987 if (ret < 0)
988 return ret;
989
990 if (in->sw->config.depth < out->sw->config.depth) {
991 tmp = min(*alloc_down, max_bw);
992 ret = usb4_dp_port_allocate_bandwidth(in, tmp);
993 if (ret)
994 return ret;
995 *alloc_down = tmp;
996 *alloc_up = 0;
997 } else {
998 tmp = min(*alloc_up, max_bw);
999 ret = usb4_dp_port_allocate_bandwidth(in, tmp);
1000 if (ret)
1001 return ret;
1002 *alloc_down = 0;
1003 *alloc_up = tmp;
1004 }
1005
1006 /* Now we can use BW mode registers to figure out the bandwidth */
1007 /* TODO: need to handle discovery too */
1008 tunnel->bw_mode = true;
1009
1010 tb_port_dbg(in, "allocated bandwidth through allocation mode %d Mb/s\n",
1011 tmp);
1012 return 0;
1013 }
1014
tb_dp_read_dprx(struct tb_tunnel * tunnel,u32 * rate,u32 * lanes,int timeout_msec)1015 static int tb_dp_read_dprx(struct tb_tunnel *tunnel, u32 *rate, u32 *lanes,
1016 int timeout_msec)
1017 {
1018 ktime_t timeout = ktime_add_ms(ktime_get(), timeout_msec);
1019 struct tb_port *in = tunnel->src_port;
1020
1021 /*
1022 * Wait for DPRX done. Normally it should be already set for
1023 * active tunnel.
1024 */
1025 do {
1026 u32 val;
1027 int ret;
1028
1029 ret = tb_port_read(in, &val, TB_CFG_PORT,
1030 in->cap_adap + DP_COMMON_CAP, 1);
1031 if (ret)
1032 return ret;
1033
1034 if (val & DP_COMMON_CAP_DPRX_DONE) {
1035 *rate = tb_dp_cap_get_rate(val);
1036 *lanes = tb_dp_cap_get_lanes(val);
1037
1038 tb_port_dbg(in, "consumed bandwidth through DPRX %d Mb/s\n",
1039 tb_dp_bandwidth(*rate, *lanes));
1040 return 0;
1041 }
1042 usleep_range(100, 150);
1043 } while (ktime_before(ktime_get(), timeout));
1044
1045 return -ETIMEDOUT;
1046 }
1047
1048 /* Read cap from tunnel DP IN */
tb_dp_read_cap(struct tb_tunnel * tunnel,unsigned int cap,u32 * rate,u32 * lanes)1049 static int tb_dp_read_cap(struct tb_tunnel *tunnel, unsigned int cap, u32 *rate,
1050 u32 *lanes)
1051 {
1052 struct tb_port *in = tunnel->src_port;
1053 u32 val;
1054 int ret;
1055
1056 switch (cap) {
1057 case DP_LOCAL_CAP:
1058 case DP_REMOTE_CAP:
1059 break;
1060
1061 default:
1062 tb_tunnel_WARN(tunnel, "invalid capability index %#x\n", cap);
1063 return -EINVAL;
1064 }
1065
1066 /*
1067 * Read from the copied remote cap so that we take into account
1068 * if capabilities were reduced during exchange.
1069 */
1070 ret = tb_port_read(in, &val, TB_CFG_PORT, in->cap_adap + cap, 1);
1071 if (ret)
1072 return ret;
1073
1074 *rate = tb_dp_cap_get_rate(val);
1075 *lanes = tb_dp_cap_get_lanes(val);
1076
1077 tb_port_dbg(in, "bandwidth from %#x capability %d Mb/s\n", cap,
1078 tb_dp_bandwidth(*rate, *lanes));
1079 return 0;
1080 }
1081
tb_dp_maximum_bandwidth(struct tb_tunnel * tunnel,int * max_up,int * max_down)1082 static int tb_dp_maximum_bandwidth(struct tb_tunnel *tunnel, int *max_up,
1083 int *max_down)
1084 {
1085 struct tb_port *in = tunnel->src_port;
1086 int ret;
1087
1088 if (!usb4_dp_port_bandwidth_mode_enabled(in))
1089 return -EOPNOTSUPP;
1090
1091 ret = tb_dp_bandwidth_mode_maximum_bandwidth(tunnel, NULL);
1092 if (ret < 0)
1093 return ret;
1094
1095 if (in->sw->config.depth < tunnel->dst_port->sw->config.depth) {
1096 *max_up = 0;
1097 *max_down = ret;
1098 } else {
1099 *max_up = ret;
1100 *max_down = 0;
1101 }
1102
1103 return 0;
1104 }
1105
tb_dp_consumed_bandwidth(struct tb_tunnel * tunnel,int * consumed_up,int * consumed_down)1106 static int tb_dp_consumed_bandwidth(struct tb_tunnel *tunnel, int *consumed_up,
1107 int *consumed_down)
1108 {
1109 struct tb_port *in = tunnel->src_port;
1110 const struct tb_switch *sw = in->sw;
1111 u32 rate = 0, lanes = 0;
1112 int ret;
1113
1114 if (tb_dp_is_usb4(sw)) {
1115 /*
1116 * On USB4 routers check if the bandwidth allocation
1117 * mode is enabled first and then read the bandwidth
1118 * through those registers.
1119 */
1120 ret = tb_dp_bandwidth_mode_consumed_bandwidth(tunnel, consumed_up,
1121 consumed_down);
1122 if (ret < 0) {
1123 if (ret != -EOPNOTSUPP)
1124 return ret;
1125 } else if (!ret) {
1126 return 0;
1127 }
1128 /*
1129 * Then see if the DPRX negotiation is ready and if yes
1130 * return that bandwidth (it may be smaller than the
1131 * reduced one). Otherwise return the remote (possibly
1132 * reduced) caps.
1133 */
1134 ret = tb_dp_read_dprx(tunnel, &rate, &lanes, 150);
1135 if (ret) {
1136 if (ret == -ETIMEDOUT)
1137 ret = tb_dp_read_cap(tunnel, DP_REMOTE_CAP,
1138 &rate, &lanes);
1139 if (ret)
1140 return ret;
1141 }
1142 } else if (sw->generation >= 2) {
1143 ret = tb_dp_read_cap(tunnel, DP_REMOTE_CAP, &rate, &lanes);
1144 if (ret)
1145 return ret;
1146 } else {
1147 /* No bandwidth management for legacy devices */
1148 *consumed_up = 0;
1149 *consumed_down = 0;
1150 return 0;
1151 }
1152
1153 if (in->sw->config.depth < tunnel->dst_port->sw->config.depth) {
1154 *consumed_up = 0;
1155 *consumed_down = tb_dp_bandwidth(rate, lanes);
1156 } else {
1157 *consumed_up = tb_dp_bandwidth(rate, lanes);
1158 *consumed_down = 0;
1159 }
1160
1161 return 0;
1162 }
1163
tb_dp_init_aux_credits(struct tb_path_hop * hop)1164 static void tb_dp_init_aux_credits(struct tb_path_hop *hop)
1165 {
1166 struct tb_port *port = hop->in_port;
1167 struct tb_switch *sw = port->sw;
1168
1169 if (tb_port_use_credit_allocation(port))
1170 hop->initial_credits = sw->min_dp_aux_credits;
1171 else
1172 hop->initial_credits = 1;
1173 }
1174
tb_dp_init_aux_path(struct tb_path * path)1175 static void tb_dp_init_aux_path(struct tb_path *path)
1176 {
1177 struct tb_path_hop *hop;
1178
1179 path->egress_fc_enable = TB_PATH_SOURCE | TB_PATH_INTERNAL;
1180 path->egress_shared_buffer = TB_PATH_NONE;
1181 path->ingress_fc_enable = TB_PATH_ALL;
1182 path->ingress_shared_buffer = TB_PATH_NONE;
1183 path->priority = 2;
1184 path->weight = 1;
1185
1186 tb_path_for_each_hop(path, hop)
1187 tb_dp_init_aux_credits(hop);
1188 }
1189
tb_dp_init_video_credits(struct tb_path_hop * hop)1190 static int tb_dp_init_video_credits(struct tb_path_hop *hop)
1191 {
1192 struct tb_port *port = hop->in_port;
1193 struct tb_switch *sw = port->sw;
1194
1195 if (tb_port_use_credit_allocation(port)) {
1196 unsigned int nfc_credits;
1197 size_t max_dp_streams;
1198
1199 tb_available_credits(port, &max_dp_streams);
1200 /*
1201 * Read the number of currently allocated NFC credits
1202 * from the lane adapter. Since we only use them for DP
1203 * tunneling we can use that to figure out how many DP
1204 * tunnels already go through the lane adapter.
1205 */
1206 nfc_credits = port->config.nfc_credits &
1207 ADP_CS_4_NFC_BUFFERS_MASK;
1208 if (nfc_credits / sw->min_dp_main_credits > max_dp_streams)
1209 return -ENOSPC;
1210
1211 hop->nfc_credits = sw->min_dp_main_credits;
1212 } else {
1213 hop->nfc_credits = min(port->total_credits - 2, 12U);
1214 }
1215
1216 return 0;
1217 }
1218
tb_dp_init_video_path(struct tb_path * path)1219 static int tb_dp_init_video_path(struct tb_path *path)
1220 {
1221 struct tb_path_hop *hop;
1222
1223 path->egress_fc_enable = TB_PATH_NONE;
1224 path->egress_shared_buffer = TB_PATH_NONE;
1225 path->ingress_fc_enable = TB_PATH_NONE;
1226 path->ingress_shared_buffer = TB_PATH_NONE;
1227 path->priority = 1;
1228 path->weight = 1;
1229
1230 tb_path_for_each_hop(path, hop) {
1231 int ret;
1232
1233 ret = tb_dp_init_video_credits(hop);
1234 if (ret)
1235 return ret;
1236 }
1237
1238 return 0;
1239 }
1240
tb_dp_dump(struct tb_tunnel * tunnel)1241 static void tb_dp_dump(struct tb_tunnel *tunnel)
1242 {
1243 struct tb_port *in, *out;
1244 u32 dp_cap, rate, lanes;
1245
1246 in = tunnel->src_port;
1247 out = tunnel->dst_port;
1248
1249 if (tb_port_read(in, &dp_cap, TB_CFG_PORT,
1250 in->cap_adap + DP_LOCAL_CAP, 1))
1251 return;
1252
1253 rate = tb_dp_cap_get_rate(dp_cap);
1254 lanes = tb_dp_cap_get_lanes(dp_cap);
1255
1256 tb_port_dbg(in, "maximum supported bandwidth %u Mb/s x%u = %u Mb/s\n",
1257 rate, lanes, tb_dp_bandwidth(rate, lanes));
1258
1259 out = tunnel->dst_port;
1260
1261 if (tb_port_read(out, &dp_cap, TB_CFG_PORT,
1262 out->cap_adap + DP_LOCAL_CAP, 1))
1263 return;
1264
1265 rate = tb_dp_cap_get_rate(dp_cap);
1266 lanes = tb_dp_cap_get_lanes(dp_cap);
1267
1268 tb_port_dbg(out, "maximum supported bandwidth %u Mb/s x%u = %u Mb/s\n",
1269 rate, lanes, tb_dp_bandwidth(rate, lanes));
1270
1271 if (tb_port_read(in, &dp_cap, TB_CFG_PORT,
1272 in->cap_adap + DP_REMOTE_CAP, 1))
1273 return;
1274
1275 rate = tb_dp_cap_get_rate(dp_cap);
1276 lanes = tb_dp_cap_get_lanes(dp_cap);
1277
1278 tb_port_dbg(in, "reduced bandwidth %u Mb/s x%u = %u Mb/s\n",
1279 rate, lanes, tb_dp_bandwidth(rate, lanes));
1280 }
1281
1282 /**
1283 * tb_tunnel_discover_dp() - Discover existing Display Port tunnels
1284 * @tb: Pointer to the domain structure
1285 * @in: DP in adapter
1286 * @alloc_hopid: Allocate HopIDs from visited ports
1287 *
1288 * If @in adapter is active, follows the tunnel to the DP out adapter
1289 * and back. Returns the discovered tunnel or %NULL if there was no
1290 * tunnel.
1291 *
1292 * Return: DP tunnel or %NULL if no tunnel found.
1293 */
tb_tunnel_discover_dp(struct tb * tb,struct tb_port * in,bool alloc_hopid)1294 struct tb_tunnel *tb_tunnel_discover_dp(struct tb *tb, struct tb_port *in,
1295 bool alloc_hopid)
1296 {
1297 struct tb_tunnel *tunnel;
1298 struct tb_port *port;
1299 struct tb_path *path;
1300
1301 if (!tb_dp_port_is_enabled(in))
1302 return NULL;
1303
1304 tunnel = tb_tunnel_alloc(tb, 3, TB_TUNNEL_DP);
1305 if (!tunnel)
1306 return NULL;
1307
1308 tunnel->init = tb_dp_init;
1309 tunnel->deinit = tb_dp_deinit;
1310 tunnel->activate = tb_dp_activate;
1311 tunnel->maximum_bandwidth = tb_dp_maximum_bandwidth;
1312 tunnel->allocated_bandwidth = tb_dp_allocated_bandwidth;
1313 tunnel->alloc_bandwidth = tb_dp_alloc_bandwidth;
1314 tunnel->consumed_bandwidth = tb_dp_consumed_bandwidth;
1315 tunnel->src_port = in;
1316
1317 path = tb_path_discover(in, TB_DP_VIDEO_HOPID, NULL, -1,
1318 &tunnel->dst_port, "Video", alloc_hopid);
1319 if (!path) {
1320 /* Just disable the DP IN port */
1321 tb_dp_port_enable(in, false);
1322 goto err_free;
1323 }
1324 tunnel->paths[TB_DP_VIDEO_PATH_OUT] = path;
1325 if (tb_dp_init_video_path(tunnel->paths[TB_DP_VIDEO_PATH_OUT]))
1326 goto err_free;
1327
1328 path = tb_path_discover(in, TB_DP_AUX_TX_HOPID, NULL, -1, NULL, "AUX TX",
1329 alloc_hopid);
1330 if (!path)
1331 goto err_deactivate;
1332 tunnel->paths[TB_DP_AUX_PATH_OUT] = path;
1333 tb_dp_init_aux_path(tunnel->paths[TB_DP_AUX_PATH_OUT]);
1334
1335 path = tb_path_discover(tunnel->dst_port, -1, in, TB_DP_AUX_RX_HOPID,
1336 &port, "AUX RX", alloc_hopid);
1337 if (!path)
1338 goto err_deactivate;
1339 tunnel->paths[TB_DP_AUX_PATH_IN] = path;
1340 tb_dp_init_aux_path(tunnel->paths[TB_DP_AUX_PATH_IN]);
1341
1342 /* Validate that the tunnel is complete */
1343 if (!tb_port_is_dpout(tunnel->dst_port)) {
1344 tb_port_warn(in, "path does not end on a DP adapter, cleaning up\n");
1345 goto err_deactivate;
1346 }
1347
1348 if (!tb_dp_port_is_enabled(tunnel->dst_port))
1349 goto err_deactivate;
1350
1351 if (!tb_dp_port_hpd_is_active(tunnel->dst_port))
1352 goto err_deactivate;
1353
1354 if (port != tunnel->src_port) {
1355 tb_tunnel_warn(tunnel, "path is not complete, cleaning up\n");
1356 goto err_deactivate;
1357 }
1358
1359 tb_dp_dump(tunnel);
1360
1361 tb_tunnel_dbg(tunnel, "discovered\n");
1362 return tunnel;
1363
1364 err_deactivate:
1365 tb_tunnel_deactivate(tunnel);
1366 err_free:
1367 tb_tunnel_free(tunnel);
1368
1369 return NULL;
1370 }
1371
1372 /**
1373 * tb_tunnel_alloc_dp() - allocate a Display Port tunnel
1374 * @tb: Pointer to the domain structure
1375 * @in: DP in adapter port
1376 * @out: DP out adapter port
1377 * @link_nr: Preferred lane adapter when the link is not bonded
1378 * @max_up: Maximum available upstream bandwidth for the DP tunnel (%0
1379 * if not limited)
1380 * @max_down: Maximum available downstream bandwidth for the DP tunnel
1381 * (%0 if not limited)
1382 *
1383 * Allocates a tunnel between @in and @out that is capable of tunneling
1384 * Display Port traffic.
1385 *
1386 * Return: Returns a tb_tunnel on success or NULL on failure.
1387 */
tb_tunnel_alloc_dp(struct tb * tb,struct tb_port * in,struct tb_port * out,int link_nr,int max_up,int max_down)1388 struct tb_tunnel *tb_tunnel_alloc_dp(struct tb *tb, struct tb_port *in,
1389 struct tb_port *out, int link_nr,
1390 int max_up, int max_down)
1391 {
1392 struct tb_tunnel *tunnel;
1393 struct tb_path **paths;
1394 struct tb_path *path;
1395
1396 if (WARN_ON(!in->cap_adap || !out->cap_adap))
1397 return NULL;
1398
1399 tunnel = tb_tunnel_alloc(tb, 3, TB_TUNNEL_DP);
1400 if (!tunnel)
1401 return NULL;
1402
1403 tunnel->init = tb_dp_init;
1404 tunnel->deinit = tb_dp_deinit;
1405 tunnel->activate = tb_dp_activate;
1406 tunnel->maximum_bandwidth = tb_dp_maximum_bandwidth;
1407 tunnel->allocated_bandwidth = tb_dp_allocated_bandwidth;
1408 tunnel->alloc_bandwidth = tb_dp_alloc_bandwidth;
1409 tunnel->consumed_bandwidth = tb_dp_consumed_bandwidth;
1410 tunnel->src_port = in;
1411 tunnel->dst_port = out;
1412 tunnel->max_up = max_up;
1413 tunnel->max_down = max_down;
1414
1415 paths = tunnel->paths;
1416
1417 path = tb_path_alloc(tb, in, TB_DP_VIDEO_HOPID, out, TB_DP_VIDEO_HOPID,
1418 link_nr, "Video");
1419 if (!path)
1420 goto err_free;
1421 tb_dp_init_video_path(path);
1422 paths[TB_DP_VIDEO_PATH_OUT] = path;
1423
1424 path = tb_path_alloc(tb, in, TB_DP_AUX_TX_HOPID, out,
1425 TB_DP_AUX_TX_HOPID, link_nr, "AUX TX");
1426 if (!path)
1427 goto err_free;
1428 tb_dp_init_aux_path(path);
1429 paths[TB_DP_AUX_PATH_OUT] = path;
1430
1431 path = tb_path_alloc(tb, out, TB_DP_AUX_RX_HOPID, in,
1432 TB_DP_AUX_RX_HOPID, link_nr, "AUX RX");
1433 if (!path)
1434 goto err_free;
1435 tb_dp_init_aux_path(path);
1436 paths[TB_DP_AUX_PATH_IN] = path;
1437
1438 return tunnel;
1439
1440 err_free:
1441 tb_tunnel_free(tunnel);
1442 return NULL;
1443 }
1444
tb_dma_available_credits(const struct tb_port * port)1445 static unsigned int tb_dma_available_credits(const struct tb_port *port)
1446 {
1447 const struct tb_switch *sw = port->sw;
1448 int credits;
1449
1450 credits = tb_available_credits(port, NULL);
1451 if (tb_acpi_may_tunnel_pcie())
1452 credits -= sw->max_pcie_credits;
1453 credits -= port->dma_credits;
1454
1455 return credits > 0 ? credits : 0;
1456 }
1457
tb_dma_reserve_credits(struct tb_path_hop * hop,unsigned int credits)1458 static int tb_dma_reserve_credits(struct tb_path_hop *hop, unsigned int credits)
1459 {
1460 struct tb_port *port = hop->in_port;
1461
1462 if (tb_port_use_credit_allocation(port)) {
1463 unsigned int available = tb_dma_available_credits(port);
1464
1465 /*
1466 * Need to have at least TB_MIN_DMA_CREDITS, otherwise
1467 * DMA path cannot be established.
1468 */
1469 if (available < TB_MIN_DMA_CREDITS)
1470 return -ENOSPC;
1471
1472 while (credits > available)
1473 credits--;
1474
1475 tb_port_dbg(port, "reserving %u credits for DMA path\n",
1476 credits);
1477
1478 port->dma_credits += credits;
1479 } else {
1480 if (tb_port_is_null(port))
1481 credits = port->bonded ? 14 : 6;
1482 else
1483 credits = min(port->total_credits, credits);
1484 }
1485
1486 hop->initial_credits = credits;
1487 return 0;
1488 }
1489
1490 /* Path from lane adapter to NHI */
tb_dma_init_rx_path(struct tb_path * path,unsigned int credits)1491 static int tb_dma_init_rx_path(struct tb_path *path, unsigned int credits)
1492 {
1493 struct tb_path_hop *hop;
1494 unsigned int i, tmp;
1495
1496 path->egress_fc_enable = TB_PATH_SOURCE | TB_PATH_INTERNAL;
1497 path->ingress_fc_enable = TB_PATH_ALL;
1498 path->egress_shared_buffer = TB_PATH_NONE;
1499 path->ingress_shared_buffer = TB_PATH_NONE;
1500 path->priority = 5;
1501 path->weight = 1;
1502 path->clear_fc = true;
1503
1504 /*
1505 * First lane adapter is the one connected to the remote host.
1506 * We don't tunnel other traffic over this link so can use all
1507 * the credits (except the ones reserved for control traffic).
1508 */
1509 hop = &path->hops[0];
1510 tmp = min(tb_usable_credits(hop->in_port), credits);
1511 hop->initial_credits = tmp;
1512 hop->in_port->dma_credits += tmp;
1513
1514 for (i = 1; i < path->path_length; i++) {
1515 int ret;
1516
1517 ret = tb_dma_reserve_credits(&path->hops[i], credits);
1518 if (ret)
1519 return ret;
1520 }
1521
1522 return 0;
1523 }
1524
1525 /* Path from NHI to lane adapter */
tb_dma_init_tx_path(struct tb_path * path,unsigned int credits)1526 static int tb_dma_init_tx_path(struct tb_path *path, unsigned int credits)
1527 {
1528 struct tb_path_hop *hop;
1529
1530 path->egress_fc_enable = TB_PATH_ALL;
1531 path->ingress_fc_enable = TB_PATH_ALL;
1532 path->egress_shared_buffer = TB_PATH_NONE;
1533 path->ingress_shared_buffer = TB_PATH_NONE;
1534 path->priority = 5;
1535 path->weight = 1;
1536 path->clear_fc = true;
1537
1538 tb_path_for_each_hop(path, hop) {
1539 int ret;
1540
1541 ret = tb_dma_reserve_credits(hop, credits);
1542 if (ret)
1543 return ret;
1544 }
1545
1546 return 0;
1547 }
1548
tb_dma_release_credits(struct tb_path_hop * hop)1549 static void tb_dma_release_credits(struct tb_path_hop *hop)
1550 {
1551 struct tb_port *port = hop->in_port;
1552
1553 if (tb_port_use_credit_allocation(port)) {
1554 port->dma_credits -= hop->initial_credits;
1555
1556 tb_port_dbg(port, "released %u DMA path credits\n",
1557 hop->initial_credits);
1558 }
1559 }
1560
tb_dma_deinit_path(struct tb_path * path)1561 static void tb_dma_deinit_path(struct tb_path *path)
1562 {
1563 struct tb_path_hop *hop;
1564
1565 tb_path_for_each_hop(path, hop)
1566 tb_dma_release_credits(hop);
1567 }
1568
tb_dma_deinit(struct tb_tunnel * tunnel)1569 static void tb_dma_deinit(struct tb_tunnel *tunnel)
1570 {
1571 int i;
1572
1573 for (i = 0; i < tunnel->npaths; i++) {
1574 if (!tunnel->paths[i])
1575 continue;
1576 tb_dma_deinit_path(tunnel->paths[i]);
1577 }
1578 }
1579
1580 /**
1581 * tb_tunnel_alloc_dma() - allocate a DMA tunnel
1582 * @tb: Pointer to the domain structure
1583 * @nhi: Host controller port
1584 * @dst: Destination null port which the other domain is connected to
1585 * @transmit_path: HopID used for transmitting packets
1586 * @transmit_ring: NHI ring number used to send packets towards the
1587 * other domain. Set to %-1 if TX path is not needed.
1588 * @receive_path: HopID used for receiving packets
1589 * @receive_ring: NHI ring number used to receive packets from the
1590 * other domain. Set to %-1 if RX path is not needed.
1591 *
1592 * Return: Returns a tb_tunnel on success or NULL on failure.
1593 */
tb_tunnel_alloc_dma(struct tb * tb,struct tb_port * nhi,struct tb_port * dst,int transmit_path,int transmit_ring,int receive_path,int receive_ring)1594 struct tb_tunnel *tb_tunnel_alloc_dma(struct tb *tb, struct tb_port *nhi,
1595 struct tb_port *dst, int transmit_path,
1596 int transmit_ring, int receive_path,
1597 int receive_ring)
1598 {
1599 struct tb_tunnel *tunnel;
1600 size_t npaths = 0, i = 0;
1601 struct tb_path *path;
1602 int credits;
1603
1604 /* Ring 0 is reserved for control channel */
1605 if (WARN_ON(!receive_ring || !transmit_ring))
1606 return NULL;
1607
1608 if (receive_ring > 0)
1609 npaths++;
1610 if (transmit_ring > 0)
1611 npaths++;
1612
1613 if (WARN_ON(!npaths))
1614 return NULL;
1615
1616 tunnel = tb_tunnel_alloc(tb, npaths, TB_TUNNEL_DMA);
1617 if (!tunnel)
1618 return NULL;
1619
1620 tunnel->src_port = nhi;
1621 tunnel->dst_port = dst;
1622 tunnel->deinit = tb_dma_deinit;
1623
1624 credits = min_not_zero(dma_credits, nhi->sw->max_dma_credits);
1625
1626 if (receive_ring > 0) {
1627 path = tb_path_alloc(tb, dst, receive_path, nhi, receive_ring, 0,
1628 "DMA RX");
1629 if (!path)
1630 goto err_free;
1631 tunnel->paths[i++] = path;
1632 if (tb_dma_init_rx_path(path, credits)) {
1633 tb_tunnel_dbg(tunnel, "not enough buffers for RX path\n");
1634 goto err_free;
1635 }
1636 }
1637
1638 if (transmit_ring > 0) {
1639 path = tb_path_alloc(tb, nhi, transmit_ring, dst, transmit_path, 0,
1640 "DMA TX");
1641 if (!path)
1642 goto err_free;
1643 tunnel->paths[i++] = path;
1644 if (tb_dma_init_tx_path(path, credits)) {
1645 tb_tunnel_dbg(tunnel, "not enough buffers for TX path\n");
1646 goto err_free;
1647 }
1648 }
1649
1650 return tunnel;
1651
1652 err_free:
1653 tb_tunnel_free(tunnel);
1654 return NULL;
1655 }
1656
1657 /**
1658 * tb_tunnel_match_dma() - Match DMA tunnel
1659 * @tunnel: Tunnel to match
1660 * @transmit_path: HopID used for transmitting packets. Pass %-1 to ignore.
1661 * @transmit_ring: NHI ring number used to send packets towards the
1662 * other domain. Pass %-1 to ignore.
1663 * @receive_path: HopID used for receiving packets. Pass %-1 to ignore.
1664 * @receive_ring: NHI ring number used to receive packets from the
1665 * other domain. Pass %-1 to ignore.
1666 *
1667 * This function can be used to match specific DMA tunnel, if there are
1668 * multiple DMA tunnels going through the same XDomain connection.
1669 * Returns true if there is match and false otherwise.
1670 */
tb_tunnel_match_dma(const struct tb_tunnel * tunnel,int transmit_path,int transmit_ring,int receive_path,int receive_ring)1671 bool tb_tunnel_match_dma(const struct tb_tunnel *tunnel, int transmit_path,
1672 int transmit_ring, int receive_path, int receive_ring)
1673 {
1674 const struct tb_path *tx_path = NULL, *rx_path = NULL;
1675 int i;
1676
1677 if (!receive_ring || !transmit_ring)
1678 return false;
1679
1680 for (i = 0; i < tunnel->npaths; i++) {
1681 const struct tb_path *path = tunnel->paths[i];
1682
1683 if (!path)
1684 continue;
1685
1686 if (tb_port_is_nhi(path->hops[0].in_port))
1687 tx_path = path;
1688 else if (tb_port_is_nhi(path->hops[path->path_length - 1].out_port))
1689 rx_path = path;
1690 }
1691
1692 if (transmit_ring > 0 || transmit_path > 0) {
1693 if (!tx_path)
1694 return false;
1695 if (transmit_ring > 0 &&
1696 (tx_path->hops[0].in_hop_index != transmit_ring))
1697 return false;
1698 if (transmit_path > 0 &&
1699 (tx_path->hops[tx_path->path_length - 1].next_hop_index != transmit_path))
1700 return false;
1701 }
1702
1703 if (receive_ring > 0 || receive_path > 0) {
1704 if (!rx_path)
1705 return false;
1706 if (receive_path > 0 &&
1707 (rx_path->hops[0].in_hop_index != receive_path))
1708 return false;
1709 if (receive_ring > 0 &&
1710 (rx_path->hops[rx_path->path_length - 1].next_hop_index != receive_ring))
1711 return false;
1712 }
1713
1714 return true;
1715 }
1716
tb_usb3_max_link_rate(struct tb_port * up,struct tb_port * down)1717 static int tb_usb3_max_link_rate(struct tb_port *up, struct tb_port *down)
1718 {
1719 int ret, up_max_rate, down_max_rate;
1720
1721 ret = usb4_usb3_port_max_link_rate(up);
1722 if (ret < 0)
1723 return ret;
1724 up_max_rate = ret;
1725
1726 ret = usb4_usb3_port_max_link_rate(down);
1727 if (ret < 0)
1728 return ret;
1729 down_max_rate = ret;
1730
1731 return min(up_max_rate, down_max_rate);
1732 }
1733
tb_usb3_init(struct tb_tunnel * tunnel)1734 static int tb_usb3_init(struct tb_tunnel *tunnel)
1735 {
1736 tb_tunnel_dbg(tunnel, "allocating initial bandwidth %d/%d Mb/s\n",
1737 tunnel->allocated_up, tunnel->allocated_down);
1738
1739 return usb4_usb3_port_allocate_bandwidth(tunnel->src_port,
1740 &tunnel->allocated_up,
1741 &tunnel->allocated_down);
1742 }
1743
tb_usb3_activate(struct tb_tunnel * tunnel,bool activate)1744 static int tb_usb3_activate(struct tb_tunnel *tunnel, bool activate)
1745 {
1746 int res;
1747
1748 res = tb_usb3_port_enable(tunnel->src_port, activate);
1749 if (res)
1750 return res;
1751
1752 if (tb_port_is_usb3_up(tunnel->dst_port))
1753 return tb_usb3_port_enable(tunnel->dst_port, activate);
1754
1755 return 0;
1756 }
1757
tb_usb3_consumed_bandwidth(struct tb_tunnel * tunnel,int * consumed_up,int * consumed_down)1758 static int tb_usb3_consumed_bandwidth(struct tb_tunnel *tunnel,
1759 int *consumed_up, int *consumed_down)
1760 {
1761 int pcie_enabled = tb_acpi_may_tunnel_pcie();
1762
1763 /*
1764 * PCIe tunneling, if enabled, affects the USB3 bandwidth so
1765 * take that it into account here.
1766 */
1767 *consumed_up = tunnel->allocated_up * (3 + pcie_enabled) / 3;
1768 *consumed_down = tunnel->allocated_down * (3 + pcie_enabled) / 3;
1769 return 0;
1770 }
1771
tb_usb3_release_unused_bandwidth(struct tb_tunnel * tunnel)1772 static int tb_usb3_release_unused_bandwidth(struct tb_tunnel *tunnel)
1773 {
1774 int ret;
1775
1776 ret = usb4_usb3_port_release_bandwidth(tunnel->src_port,
1777 &tunnel->allocated_up,
1778 &tunnel->allocated_down);
1779 if (ret)
1780 return ret;
1781
1782 tb_tunnel_dbg(tunnel, "decreased bandwidth allocation to %d/%d Mb/s\n",
1783 tunnel->allocated_up, tunnel->allocated_down);
1784 return 0;
1785 }
1786
tb_usb3_reclaim_available_bandwidth(struct tb_tunnel * tunnel,int * available_up,int * available_down)1787 static void tb_usb3_reclaim_available_bandwidth(struct tb_tunnel *tunnel,
1788 int *available_up,
1789 int *available_down)
1790 {
1791 int ret, max_rate, allocate_up, allocate_down;
1792
1793 ret = usb4_usb3_port_actual_link_rate(tunnel->src_port);
1794 if (ret < 0) {
1795 tb_tunnel_warn(tunnel, "failed to read actual link rate\n");
1796 return;
1797 } else if (!ret) {
1798 /* Use maximum link rate if the link valid is not set */
1799 ret = tb_usb3_max_link_rate(tunnel->dst_port, tunnel->src_port);
1800 if (ret < 0) {
1801 tb_tunnel_warn(tunnel, "failed to read maximum link rate\n");
1802 return;
1803 }
1804 }
1805
1806 /*
1807 * 90% of the max rate can be allocated for isochronous
1808 * transfers.
1809 */
1810 max_rate = ret * 90 / 100;
1811
1812 /* No need to reclaim if already at maximum */
1813 if (tunnel->allocated_up >= max_rate &&
1814 tunnel->allocated_down >= max_rate)
1815 return;
1816
1817 /* Don't go lower than what is already allocated */
1818 allocate_up = min(max_rate, *available_up);
1819 if (allocate_up < tunnel->allocated_up)
1820 allocate_up = tunnel->allocated_up;
1821
1822 allocate_down = min(max_rate, *available_down);
1823 if (allocate_down < tunnel->allocated_down)
1824 allocate_down = tunnel->allocated_down;
1825
1826 /* If no changes no need to do more */
1827 if (allocate_up == tunnel->allocated_up &&
1828 allocate_down == tunnel->allocated_down)
1829 return;
1830
1831 ret = usb4_usb3_port_allocate_bandwidth(tunnel->src_port, &allocate_up,
1832 &allocate_down);
1833 if (ret) {
1834 tb_tunnel_info(tunnel, "failed to allocate bandwidth\n");
1835 return;
1836 }
1837
1838 tunnel->allocated_up = allocate_up;
1839 *available_up -= tunnel->allocated_up;
1840
1841 tunnel->allocated_down = allocate_down;
1842 *available_down -= tunnel->allocated_down;
1843
1844 tb_tunnel_dbg(tunnel, "increased bandwidth allocation to %d/%d Mb/s\n",
1845 tunnel->allocated_up, tunnel->allocated_down);
1846 }
1847
tb_usb3_init_credits(struct tb_path_hop * hop)1848 static void tb_usb3_init_credits(struct tb_path_hop *hop)
1849 {
1850 struct tb_port *port = hop->in_port;
1851 struct tb_switch *sw = port->sw;
1852 unsigned int credits;
1853
1854 if (tb_port_use_credit_allocation(port)) {
1855 credits = sw->max_usb3_credits;
1856 } else {
1857 if (tb_port_is_null(port))
1858 credits = port->bonded ? 32 : 16;
1859 else
1860 credits = 7;
1861 }
1862
1863 hop->initial_credits = credits;
1864 }
1865
tb_usb3_init_path(struct tb_path * path)1866 static void tb_usb3_init_path(struct tb_path *path)
1867 {
1868 struct tb_path_hop *hop;
1869
1870 path->egress_fc_enable = TB_PATH_SOURCE | TB_PATH_INTERNAL;
1871 path->egress_shared_buffer = TB_PATH_NONE;
1872 path->ingress_fc_enable = TB_PATH_ALL;
1873 path->ingress_shared_buffer = TB_PATH_NONE;
1874 path->priority = 3;
1875 path->weight = 3;
1876 path->drop_packages = 0;
1877
1878 tb_path_for_each_hop(path, hop)
1879 tb_usb3_init_credits(hop);
1880 }
1881
1882 /**
1883 * tb_tunnel_discover_usb3() - Discover existing USB3 tunnels
1884 * @tb: Pointer to the domain structure
1885 * @down: USB3 downstream adapter
1886 * @alloc_hopid: Allocate HopIDs from visited ports
1887 *
1888 * If @down adapter is active, follows the tunnel to the USB3 upstream
1889 * adapter and back. Returns the discovered tunnel or %NULL if there was
1890 * no tunnel.
1891 */
tb_tunnel_discover_usb3(struct tb * tb,struct tb_port * down,bool alloc_hopid)1892 struct tb_tunnel *tb_tunnel_discover_usb3(struct tb *tb, struct tb_port *down,
1893 bool alloc_hopid)
1894 {
1895 struct tb_tunnel *tunnel;
1896 struct tb_path *path;
1897
1898 if (!tb_usb3_port_is_enabled(down))
1899 return NULL;
1900
1901 tunnel = tb_tunnel_alloc(tb, 2, TB_TUNNEL_USB3);
1902 if (!tunnel)
1903 return NULL;
1904
1905 tunnel->activate = tb_usb3_activate;
1906 tunnel->src_port = down;
1907
1908 /*
1909 * Discover both paths even if they are not complete. We will
1910 * clean them up by calling tb_tunnel_deactivate() below in that
1911 * case.
1912 */
1913 path = tb_path_discover(down, TB_USB3_HOPID, NULL, -1,
1914 &tunnel->dst_port, "USB3 Down", alloc_hopid);
1915 if (!path) {
1916 /* Just disable the downstream port */
1917 tb_usb3_port_enable(down, false);
1918 goto err_free;
1919 }
1920 tunnel->paths[TB_USB3_PATH_DOWN] = path;
1921 tb_usb3_init_path(tunnel->paths[TB_USB3_PATH_DOWN]);
1922
1923 path = tb_path_discover(tunnel->dst_port, -1, down, TB_USB3_HOPID, NULL,
1924 "USB3 Up", alloc_hopid);
1925 if (!path)
1926 goto err_deactivate;
1927 tunnel->paths[TB_USB3_PATH_UP] = path;
1928 tb_usb3_init_path(tunnel->paths[TB_USB3_PATH_UP]);
1929
1930 /* Validate that the tunnel is complete */
1931 if (!tb_port_is_usb3_up(tunnel->dst_port)) {
1932 tb_port_warn(tunnel->dst_port,
1933 "path does not end on an USB3 adapter, cleaning up\n");
1934 goto err_deactivate;
1935 }
1936
1937 if (down != tunnel->src_port) {
1938 tb_tunnel_warn(tunnel, "path is not complete, cleaning up\n");
1939 goto err_deactivate;
1940 }
1941
1942 if (!tb_usb3_port_is_enabled(tunnel->dst_port)) {
1943 tb_tunnel_warn(tunnel,
1944 "tunnel is not fully activated, cleaning up\n");
1945 goto err_deactivate;
1946 }
1947
1948 if (!tb_route(down->sw)) {
1949 int ret;
1950
1951 /*
1952 * Read the initial bandwidth allocation for the first
1953 * hop tunnel.
1954 */
1955 ret = usb4_usb3_port_allocated_bandwidth(down,
1956 &tunnel->allocated_up, &tunnel->allocated_down);
1957 if (ret)
1958 goto err_deactivate;
1959
1960 tb_tunnel_dbg(tunnel, "currently allocated bandwidth %d/%d Mb/s\n",
1961 tunnel->allocated_up, tunnel->allocated_down);
1962
1963 tunnel->init = tb_usb3_init;
1964 tunnel->consumed_bandwidth = tb_usb3_consumed_bandwidth;
1965 tunnel->release_unused_bandwidth =
1966 tb_usb3_release_unused_bandwidth;
1967 tunnel->reclaim_available_bandwidth =
1968 tb_usb3_reclaim_available_bandwidth;
1969 }
1970
1971 tb_tunnel_dbg(tunnel, "discovered\n");
1972 return tunnel;
1973
1974 err_deactivate:
1975 tb_tunnel_deactivate(tunnel);
1976 err_free:
1977 tb_tunnel_free(tunnel);
1978
1979 return NULL;
1980 }
1981
1982 /**
1983 * tb_tunnel_alloc_usb3() - allocate a USB3 tunnel
1984 * @tb: Pointer to the domain structure
1985 * @up: USB3 upstream adapter port
1986 * @down: USB3 downstream adapter port
1987 * @max_up: Maximum available upstream bandwidth for the USB3 tunnel (%0
1988 * if not limited).
1989 * @max_down: Maximum available downstream bandwidth for the USB3 tunnel
1990 * (%0 if not limited).
1991 *
1992 * Allocate an USB3 tunnel. The ports must be of type @TB_TYPE_USB3_UP and
1993 * @TB_TYPE_USB3_DOWN.
1994 *
1995 * Return: Returns a tb_tunnel on success or %NULL on failure.
1996 */
tb_tunnel_alloc_usb3(struct tb * tb,struct tb_port * up,struct tb_port * down,int max_up,int max_down)1997 struct tb_tunnel *tb_tunnel_alloc_usb3(struct tb *tb, struct tb_port *up,
1998 struct tb_port *down, int max_up,
1999 int max_down)
2000 {
2001 struct tb_tunnel *tunnel;
2002 struct tb_path *path;
2003 int max_rate = 0;
2004
2005 /*
2006 * Check that we have enough bandwidth available for the new
2007 * USB3 tunnel.
2008 */
2009 if (max_up > 0 || max_down > 0) {
2010 max_rate = tb_usb3_max_link_rate(down, up);
2011 if (max_rate < 0)
2012 return NULL;
2013
2014 /* Only 90% can be allocated for USB3 isochronous transfers */
2015 max_rate = max_rate * 90 / 100;
2016 tb_port_dbg(up, "required bandwidth for USB3 tunnel %d Mb/s\n",
2017 max_rate);
2018
2019 if (max_rate > max_up || max_rate > max_down) {
2020 tb_port_warn(up, "not enough bandwidth for USB3 tunnel\n");
2021 return NULL;
2022 }
2023 }
2024
2025 tunnel = tb_tunnel_alloc(tb, 2, TB_TUNNEL_USB3);
2026 if (!tunnel)
2027 return NULL;
2028
2029 tunnel->activate = tb_usb3_activate;
2030 tunnel->src_port = down;
2031 tunnel->dst_port = up;
2032 tunnel->max_up = max_up;
2033 tunnel->max_down = max_down;
2034
2035 path = tb_path_alloc(tb, down, TB_USB3_HOPID, up, TB_USB3_HOPID, 0,
2036 "USB3 Down");
2037 if (!path) {
2038 tb_tunnel_free(tunnel);
2039 return NULL;
2040 }
2041 tb_usb3_init_path(path);
2042 tunnel->paths[TB_USB3_PATH_DOWN] = path;
2043
2044 path = tb_path_alloc(tb, up, TB_USB3_HOPID, down, TB_USB3_HOPID, 0,
2045 "USB3 Up");
2046 if (!path) {
2047 tb_tunnel_free(tunnel);
2048 return NULL;
2049 }
2050 tb_usb3_init_path(path);
2051 tunnel->paths[TB_USB3_PATH_UP] = path;
2052
2053 if (!tb_route(down->sw)) {
2054 tunnel->allocated_up = max_rate;
2055 tunnel->allocated_down = max_rate;
2056
2057 tunnel->init = tb_usb3_init;
2058 tunnel->consumed_bandwidth = tb_usb3_consumed_bandwidth;
2059 tunnel->release_unused_bandwidth =
2060 tb_usb3_release_unused_bandwidth;
2061 tunnel->reclaim_available_bandwidth =
2062 tb_usb3_reclaim_available_bandwidth;
2063 }
2064
2065 return tunnel;
2066 }
2067
2068 /**
2069 * tb_tunnel_free() - free a tunnel
2070 * @tunnel: Tunnel to be freed
2071 *
2072 * Frees a tunnel. The tunnel does not need to be deactivated.
2073 */
tb_tunnel_free(struct tb_tunnel * tunnel)2074 void tb_tunnel_free(struct tb_tunnel *tunnel)
2075 {
2076 int i;
2077
2078 if (!tunnel)
2079 return;
2080
2081 if (tunnel->deinit)
2082 tunnel->deinit(tunnel);
2083
2084 for (i = 0; i < tunnel->npaths; i++) {
2085 if (tunnel->paths[i])
2086 tb_path_free(tunnel->paths[i]);
2087 }
2088
2089 kfree(tunnel->paths);
2090 kfree(tunnel);
2091 }
2092
2093 /**
2094 * tb_tunnel_is_invalid - check whether an activated path is still valid
2095 * @tunnel: Tunnel to check
2096 */
tb_tunnel_is_invalid(struct tb_tunnel * tunnel)2097 bool tb_tunnel_is_invalid(struct tb_tunnel *tunnel)
2098 {
2099 int i;
2100
2101 for (i = 0; i < tunnel->npaths; i++) {
2102 WARN_ON(!tunnel->paths[i]->activated);
2103 if (tb_path_is_invalid(tunnel->paths[i]))
2104 return true;
2105 }
2106
2107 return false;
2108 }
2109
2110 /**
2111 * tb_tunnel_restart() - activate a tunnel after a hardware reset
2112 * @tunnel: Tunnel to restart
2113 *
2114 * Return: 0 on success and negative errno in case if failure
2115 */
tb_tunnel_restart(struct tb_tunnel * tunnel)2116 int tb_tunnel_restart(struct tb_tunnel *tunnel)
2117 {
2118 int res, i;
2119
2120 tb_tunnel_dbg(tunnel, "activating\n");
2121
2122 /*
2123 * Make sure all paths are properly disabled before enabling
2124 * them again.
2125 */
2126 for (i = 0; i < tunnel->npaths; i++) {
2127 if (tunnel->paths[i]->activated) {
2128 tb_path_deactivate(tunnel->paths[i]);
2129 tunnel->paths[i]->activated = false;
2130 }
2131 }
2132
2133 if (tunnel->init) {
2134 res = tunnel->init(tunnel);
2135 if (res)
2136 return res;
2137 }
2138
2139 for (i = 0; i < tunnel->npaths; i++) {
2140 res = tb_path_activate(tunnel->paths[i]);
2141 if (res)
2142 goto err;
2143 }
2144
2145 if (tunnel->activate) {
2146 res = tunnel->activate(tunnel, true);
2147 if (res)
2148 goto err;
2149 }
2150
2151 return 0;
2152
2153 err:
2154 tb_tunnel_warn(tunnel, "activation failed\n");
2155 tb_tunnel_deactivate(tunnel);
2156 return res;
2157 }
2158
2159 /**
2160 * tb_tunnel_activate() - activate a tunnel
2161 * @tunnel: Tunnel to activate
2162 *
2163 * Return: Returns 0 on success or an error code on failure.
2164 */
tb_tunnel_activate(struct tb_tunnel * tunnel)2165 int tb_tunnel_activate(struct tb_tunnel *tunnel)
2166 {
2167 int i;
2168
2169 for (i = 0; i < tunnel->npaths; i++) {
2170 if (tunnel->paths[i]->activated) {
2171 tb_tunnel_WARN(tunnel,
2172 "trying to activate an already activated tunnel\n");
2173 return -EINVAL;
2174 }
2175 }
2176
2177 return tb_tunnel_restart(tunnel);
2178 }
2179
2180 /**
2181 * tb_tunnel_deactivate() - deactivate a tunnel
2182 * @tunnel: Tunnel to deactivate
2183 */
tb_tunnel_deactivate(struct tb_tunnel * tunnel)2184 void tb_tunnel_deactivate(struct tb_tunnel *tunnel)
2185 {
2186 int i;
2187
2188 tb_tunnel_dbg(tunnel, "deactivating\n");
2189
2190 if (tunnel->activate)
2191 tunnel->activate(tunnel, false);
2192
2193 for (i = 0; i < tunnel->npaths; i++) {
2194 if (tunnel->paths[i] && tunnel->paths[i]->activated)
2195 tb_path_deactivate(tunnel->paths[i]);
2196 }
2197 }
2198
2199 /**
2200 * tb_tunnel_port_on_path() - Does the tunnel go through port
2201 * @tunnel: Tunnel to check
2202 * @port: Port to check
2203 *
2204 * Returns true if @tunnel goes through @port (direction does not matter),
2205 * false otherwise.
2206 */
tb_tunnel_port_on_path(const struct tb_tunnel * tunnel,const struct tb_port * port)2207 bool tb_tunnel_port_on_path(const struct tb_tunnel *tunnel,
2208 const struct tb_port *port)
2209 {
2210 int i;
2211
2212 for (i = 0; i < tunnel->npaths; i++) {
2213 if (!tunnel->paths[i])
2214 continue;
2215
2216 if (tb_path_port_on_path(tunnel->paths[i], port))
2217 return true;
2218 }
2219
2220 return false;
2221 }
2222
tb_tunnel_is_active(const struct tb_tunnel * tunnel)2223 static bool tb_tunnel_is_active(const struct tb_tunnel *tunnel)
2224 {
2225 int i;
2226
2227 for (i = 0; i < tunnel->npaths; i++) {
2228 if (!tunnel->paths[i])
2229 return false;
2230 if (!tunnel->paths[i]->activated)
2231 return false;
2232 }
2233
2234 return true;
2235 }
2236
2237 /**
2238 * tb_tunnel_maximum_bandwidth() - Return maximum possible bandwidth
2239 * @tunnel: Tunnel to check
2240 * @max_up: Maximum upstream bandwidth in Mb/s
2241 * @max_down: Maximum downstream bandwidth in Mb/s
2242 *
2243 * Returns maximum possible bandwidth this tunnel can go if not limited
2244 * by other bandwidth clients. If the tunnel does not support this
2245 * returns %-EOPNOTSUPP.
2246 */
tb_tunnel_maximum_bandwidth(struct tb_tunnel * tunnel,int * max_up,int * max_down)2247 int tb_tunnel_maximum_bandwidth(struct tb_tunnel *tunnel, int *max_up,
2248 int *max_down)
2249 {
2250 if (!tb_tunnel_is_active(tunnel))
2251 return -EINVAL;
2252
2253 if (tunnel->maximum_bandwidth)
2254 return tunnel->maximum_bandwidth(tunnel, max_up, max_down);
2255 return -EOPNOTSUPP;
2256 }
2257
2258 /**
2259 * tb_tunnel_allocated_bandwidth() - Return bandwidth allocated for the tunnel
2260 * @tunnel: Tunnel to check
2261 * @allocated_up: Currently allocated upstream bandwidth in Mb/s is stored here
2262 * @allocated_down: Currently allocated downstream bandwidth in Mb/s is
2263 * stored here
2264 *
2265 * Returns the bandwidth allocated for the tunnel. This may be higher
2266 * than what the tunnel actually consumes.
2267 */
tb_tunnel_allocated_bandwidth(struct tb_tunnel * tunnel,int * allocated_up,int * allocated_down)2268 int tb_tunnel_allocated_bandwidth(struct tb_tunnel *tunnel, int *allocated_up,
2269 int *allocated_down)
2270 {
2271 if (!tb_tunnel_is_active(tunnel))
2272 return -EINVAL;
2273
2274 if (tunnel->allocated_bandwidth)
2275 return tunnel->allocated_bandwidth(tunnel, allocated_up,
2276 allocated_down);
2277 return -EOPNOTSUPP;
2278 }
2279
2280 /**
2281 * tb_tunnel_alloc_bandwidth() - Change tunnel bandwidth allocation
2282 * @tunnel: Tunnel whose bandwidth allocation to change
2283 * @alloc_up: New upstream bandwidth in Mb/s
2284 * @alloc_down: New downstream bandwidth in Mb/s
2285 *
2286 * Tries to change tunnel bandwidth allocation. If succeeds returns %0
2287 * and updates @alloc_up and @alloc_down to that was actually allocated
2288 * (it may not be the same as passed originally). Returns negative errno
2289 * in case of failure.
2290 */
tb_tunnel_alloc_bandwidth(struct tb_tunnel * tunnel,int * alloc_up,int * alloc_down)2291 int tb_tunnel_alloc_bandwidth(struct tb_tunnel *tunnel, int *alloc_up,
2292 int *alloc_down)
2293 {
2294 if (!tb_tunnel_is_active(tunnel))
2295 return -EINVAL;
2296
2297 if (tunnel->alloc_bandwidth)
2298 return tunnel->alloc_bandwidth(tunnel, alloc_up, alloc_down);
2299
2300 return -EOPNOTSUPP;
2301 }
2302
2303 /**
2304 * tb_tunnel_consumed_bandwidth() - Return bandwidth consumed by the tunnel
2305 * @tunnel: Tunnel to check
2306 * @consumed_up: Consumed bandwidth in Mb/s from @dst_port to @src_port.
2307 * Can be %NULL.
2308 * @consumed_down: Consumed bandwidth in Mb/s from @src_port to @dst_port.
2309 * Can be %NULL.
2310 *
2311 * Stores the amount of isochronous bandwidth @tunnel consumes in
2312 * @consumed_up and @consumed_down. In case of success returns %0,
2313 * negative errno otherwise.
2314 */
tb_tunnel_consumed_bandwidth(struct tb_tunnel * tunnel,int * consumed_up,int * consumed_down)2315 int tb_tunnel_consumed_bandwidth(struct tb_tunnel *tunnel, int *consumed_up,
2316 int *consumed_down)
2317 {
2318 int up_bw = 0, down_bw = 0;
2319
2320 if (!tb_tunnel_is_active(tunnel))
2321 goto out;
2322
2323 if (tunnel->consumed_bandwidth) {
2324 int ret;
2325
2326 ret = tunnel->consumed_bandwidth(tunnel, &up_bw, &down_bw);
2327 if (ret)
2328 return ret;
2329
2330 tb_tunnel_dbg(tunnel, "consumed bandwidth %d/%d Mb/s\n", up_bw,
2331 down_bw);
2332 }
2333
2334 out:
2335 if (consumed_up)
2336 *consumed_up = up_bw;
2337 if (consumed_down)
2338 *consumed_down = down_bw;
2339
2340 return 0;
2341 }
2342
2343 /**
2344 * tb_tunnel_release_unused_bandwidth() - Release unused bandwidth
2345 * @tunnel: Tunnel whose unused bandwidth to release
2346 *
2347 * If tunnel supports dynamic bandwidth management (USB3 tunnels at the
2348 * moment) this function makes it to release all the unused bandwidth.
2349 *
2350 * Returns %0 in case of success and negative errno otherwise.
2351 */
tb_tunnel_release_unused_bandwidth(struct tb_tunnel * tunnel)2352 int tb_tunnel_release_unused_bandwidth(struct tb_tunnel *tunnel)
2353 {
2354 if (!tb_tunnel_is_active(tunnel))
2355 return 0;
2356
2357 if (tunnel->release_unused_bandwidth) {
2358 int ret;
2359
2360 ret = tunnel->release_unused_bandwidth(tunnel);
2361 if (ret)
2362 return ret;
2363 }
2364
2365 return 0;
2366 }
2367
2368 /**
2369 * tb_tunnel_reclaim_available_bandwidth() - Reclaim available bandwidth
2370 * @tunnel: Tunnel reclaiming available bandwidth
2371 * @available_up: Available upstream bandwidth (in Mb/s)
2372 * @available_down: Available downstream bandwidth (in Mb/s)
2373 *
2374 * Reclaims bandwidth from @available_up and @available_down and updates
2375 * the variables accordingly (e.g decreases both according to what was
2376 * reclaimed by the tunnel). If nothing was reclaimed the values are
2377 * kept as is.
2378 */
tb_tunnel_reclaim_available_bandwidth(struct tb_tunnel * tunnel,int * available_up,int * available_down)2379 void tb_tunnel_reclaim_available_bandwidth(struct tb_tunnel *tunnel,
2380 int *available_up,
2381 int *available_down)
2382 {
2383 if (!tb_tunnel_is_active(tunnel))
2384 return;
2385
2386 if (tunnel->reclaim_available_bandwidth)
2387 tunnel->reclaim_available_bandwidth(tunnel, available_up,
2388 available_down);
2389 }
2390