1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Data Object Exchange
4 * PCIe r6.0, sec 6.30 DOE
5 *
6 * Copyright (C) 2021 Huawei
7 * Jonathan Cameron <Jonathan.Cameron@huawei.com>
8 *
9 * Copyright (C) 2022 Intel Corporation
10 * Ira Weiny <ira.weiny@intel.com>
11 */
12
13 #define dev_fmt(fmt) "DOE: " fmt
14
15 #include <linux/bitfield.h>
16 #include <linux/delay.h>
17 #include <linux/jiffies.h>
18 #include <linux/mutex.h>
19 #include <linux/pci.h>
20 #include <linux/pci-doe.h>
21 #include <linux/workqueue.h>
22
23 #define PCI_DOE_PROTOCOL_DISCOVERY 0
24
25 /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */
26 #define PCI_DOE_TIMEOUT HZ
27 #define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128)
28
29 #define PCI_DOE_FLAG_CANCEL 0
30 #define PCI_DOE_FLAG_DEAD 1
31
32 /* Max data object length is 2^18 dwords */
33 #define PCI_DOE_MAX_LENGTH (1 << 18)
34
35 /**
36 * struct pci_doe_mb - State for a single DOE mailbox
37 *
38 * This state is used to manage a single DOE mailbox capability. All fields
39 * should be considered opaque to the consumers and the structure passed into
40 * the helpers below after being created by devm_pci_doe_create()
41 *
42 * @pdev: PCI device this mailbox belongs to
43 * @cap_offset: Capability offset
44 * @prots: Array of protocols supported (encoded as long values)
45 * @wq: Wait queue for work item
46 * @work_queue: Queue of pci_doe_work items
47 * @flags: Bit array of PCI_DOE_FLAG_* flags
48 */
49 struct pci_doe_mb {
50 struct pci_dev *pdev;
51 u16 cap_offset;
52 struct xarray prots;
53
54 wait_queue_head_t wq;
55 struct workqueue_struct *work_queue;
56 unsigned long flags;
57 };
58
pci_doe_wait(struct pci_doe_mb * doe_mb,unsigned long timeout)59 static int pci_doe_wait(struct pci_doe_mb *doe_mb, unsigned long timeout)
60 {
61 if (wait_event_timeout(doe_mb->wq,
62 test_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags),
63 timeout))
64 return -EIO;
65 return 0;
66 }
67
pci_doe_write_ctrl(struct pci_doe_mb * doe_mb,u32 val)68 static void pci_doe_write_ctrl(struct pci_doe_mb *doe_mb, u32 val)
69 {
70 struct pci_dev *pdev = doe_mb->pdev;
71 int offset = doe_mb->cap_offset;
72
73 pci_write_config_dword(pdev, offset + PCI_DOE_CTRL, val);
74 }
75
pci_doe_abort(struct pci_doe_mb * doe_mb)76 static int pci_doe_abort(struct pci_doe_mb *doe_mb)
77 {
78 struct pci_dev *pdev = doe_mb->pdev;
79 int offset = doe_mb->cap_offset;
80 unsigned long timeout_jiffies;
81
82 pci_dbg(pdev, "[%x] Issuing Abort\n", offset);
83
84 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
85 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_ABORT);
86
87 do {
88 int rc;
89 u32 val;
90
91 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
92 if (rc)
93 return rc;
94 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
95
96 /* Abort success! */
97 if (!FIELD_GET(PCI_DOE_STATUS_ERROR, val) &&
98 !FIELD_GET(PCI_DOE_STATUS_BUSY, val))
99 return 0;
100
101 } while (!time_after(jiffies, timeout_jiffies));
102
103 /* Abort has timed out and the MB is dead */
104 pci_err(pdev, "[%x] ABORT timed out\n", offset);
105 return -EIO;
106 }
107
pci_doe_send_req(struct pci_doe_mb * doe_mb,struct pci_doe_task * task)108 static int pci_doe_send_req(struct pci_doe_mb *doe_mb,
109 struct pci_doe_task *task)
110 {
111 struct pci_dev *pdev = doe_mb->pdev;
112 int offset = doe_mb->cap_offset;
113 size_t length;
114 u32 val;
115 int i;
116
117 /*
118 * Check the DOE busy bit is not set. If it is set, this could indicate
119 * someone other than Linux (e.g. firmware) is using the mailbox. Note
120 * it is expected that firmware and OS will negotiate access rights via
121 * an, as yet to be defined, method.
122 */
123 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
124 if (FIELD_GET(PCI_DOE_STATUS_BUSY, val))
125 return -EBUSY;
126
127 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
128 return -EIO;
129
130 /* Length is 2 DW of header + length of payload in DW */
131 length = 2 + task->request_pl_sz / sizeof(u32);
132 if (length > PCI_DOE_MAX_LENGTH)
133 return -EIO;
134 if (length == PCI_DOE_MAX_LENGTH)
135 length = 0;
136
137 /* Write DOE Header */
138 val = FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_VID, task->prot.vid) |
139 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, task->prot.type);
140 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE, val);
141 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
142 FIELD_PREP(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH,
143 length));
144 for (i = 0; i < task->request_pl_sz / sizeof(u32); i++)
145 pci_write_config_dword(pdev, offset + PCI_DOE_WRITE,
146 task->request_pl[i]);
147
148 pci_doe_write_ctrl(doe_mb, PCI_DOE_CTRL_GO);
149
150 return 0;
151 }
152
pci_doe_data_obj_ready(struct pci_doe_mb * doe_mb)153 static bool pci_doe_data_obj_ready(struct pci_doe_mb *doe_mb)
154 {
155 struct pci_dev *pdev = doe_mb->pdev;
156 int offset = doe_mb->cap_offset;
157 u32 val;
158
159 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
160 if (FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val))
161 return true;
162 return false;
163 }
164
pci_doe_recv_resp(struct pci_doe_mb * doe_mb,struct pci_doe_task * task)165 static int pci_doe_recv_resp(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
166 {
167 struct pci_dev *pdev = doe_mb->pdev;
168 int offset = doe_mb->cap_offset;
169 size_t length, payload_length;
170 u32 val;
171 int i;
172
173 /* Read the first dword to get the protocol */
174 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
175 if ((FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val) != task->prot.vid) ||
176 (FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val) != task->prot.type)) {
177 dev_err_ratelimited(&pdev->dev, "[%x] expected [VID, Protocol] = [%04x, %02x], got [%04x, %02x]\n",
178 doe_mb->cap_offset, task->prot.vid, task->prot.type,
179 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_VID, val),
180 FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_1_TYPE, val));
181 return -EIO;
182 }
183
184 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
185 /* Read the second dword to get the length */
186 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
187 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
188
189 length = FIELD_GET(PCI_DOE_DATA_OBJECT_HEADER_2_LENGTH, val);
190 /* A value of 0x0 indicates max data object length */
191 if (!length)
192 length = PCI_DOE_MAX_LENGTH;
193 if (length < 2)
194 return -EIO;
195
196 /* First 2 dwords have already been read */
197 length -= 2;
198 payload_length = min(length, task->response_pl_sz / sizeof(u32));
199 /* Read the rest of the response payload */
200 for (i = 0; i < payload_length; i++) {
201 pci_read_config_dword(pdev, offset + PCI_DOE_READ,
202 &task->response_pl[i]);
203 /* Prior to the last ack, ensure Data Object Ready */
204 if (i == (payload_length - 1) && !pci_doe_data_obj_ready(doe_mb))
205 return -EIO;
206 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
207 }
208
209 /* Flush excess length */
210 for (; i < length; i++) {
211 pci_read_config_dword(pdev, offset + PCI_DOE_READ, &val);
212 pci_write_config_dword(pdev, offset + PCI_DOE_READ, 0);
213 }
214
215 /* Final error check to pick up on any since Data Object Ready */
216 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
217 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val))
218 return -EIO;
219
220 return min(length, task->response_pl_sz / sizeof(u32)) * sizeof(u32);
221 }
222
signal_task_complete(struct pci_doe_task * task,int rv)223 static void signal_task_complete(struct pci_doe_task *task, int rv)
224 {
225 task->rv = rv;
226 task->complete(task);
227 }
228
signal_task_abort(struct pci_doe_task * task,int rv)229 static void signal_task_abort(struct pci_doe_task *task, int rv)
230 {
231 struct pci_doe_mb *doe_mb = task->doe_mb;
232 struct pci_dev *pdev = doe_mb->pdev;
233
234 if (pci_doe_abort(doe_mb)) {
235 /*
236 * If the device can't process an abort; set the mailbox dead
237 * - no more submissions
238 */
239 pci_err(pdev, "[%x] Abort failed marking mailbox dead\n",
240 doe_mb->cap_offset);
241 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
242 }
243 signal_task_complete(task, rv);
244 }
245
doe_statemachine_work(struct work_struct * work)246 static void doe_statemachine_work(struct work_struct *work)
247 {
248 struct pci_doe_task *task = container_of(work, struct pci_doe_task,
249 work);
250 struct pci_doe_mb *doe_mb = task->doe_mb;
251 struct pci_dev *pdev = doe_mb->pdev;
252 int offset = doe_mb->cap_offset;
253 unsigned long timeout_jiffies;
254 u32 val;
255 int rc;
256
257 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags)) {
258 signal_task_complete(task, -EIO);
259 return;
260 }
261
262 /* Send request */
263 rc = pci_doe_send_req(doe_mb, task);
264 if (rc) {
265 /*
266 * The specification does not provide any guidance on how to
267 * resolve conflicting requests from other entities.
268 * Furthermore, it is likely that busy will not be detected
269 * most of the time. Flag any detection of status busy with an
270 * error.
271 */
272 if (rc == -EBUSY)
273 dev_err_ratelimited(&pdev->dev, "[%x] busy detected; another entity is sending conflicting requests\n",
274 offset);
275 signal_task_abort(task, rc);
276 return;
277 }
278
279 timeout_jiffies = jiffies + PCI_DOE_TIMEOUT;
280 /* Poll for response */
281 retry_resp:
282 pci_read_config_dword(pdev, offset + PCI_DOE_STATUS, &val);
283 if (FIELD_GET(PCI_DOE_STATUS_ERROR, val)) {
284 signal_task_abort(task, -EIO);
285 return;
286 }
287
288 if (!FIELD_GET(PCI_DOE_STATUS_DATA_OBJECT_READY, val)) {
289 if (time_after(jiffies, timeout_jiffies)) {
290 signal_task_abort(task, -EIO);
291 return;
292 }
293 rc = pci_doe_wait(doe_mb, PCI_DOE_POLL_INTERVAL);
294 if (rc) {
295 signal_task_abort(task, rc);
296 return;
297 }
298 goto retry_resp;
299 }
300
301 rc = pci_doe_recv_resp(doe_mb, task);
302 if (rc < 0) {
303 signal_task_abort(task, rc);
304 return;
305 }
306
307 signal_task_complete(task, rc);
308 }
309
pci_doe_task_complete(struct pci_doe_task * task)310 static void pci_doe_task_complete(struct pci_doe_task *task)
311 {
312 complete(task->private);
313 }
314
pci_doe_discovery(struct pci_doe_mb * doe_mb,u8 * index,u16 * vid,u8 * protocol)315 static int pci_doe_discovery(struct pci_doe_mb *doe_mb, u8 *index, u16 *vid,
316 u8 *protocol)
317 {
318 u32 request_pl = FIELD_PREP(PCI_DOE_DATA_OBJECT_DISC_REQ_3_INDEX,
319 *index);
320 u32 response_pl;
321 DECLARE_COMPLETION_ONSTACK(c);
322 struct pci_doe_task task = {
323 .prot.vid = PCI_VENDOR_ID_PCI_SIG,
324 .prot.type = PCI_DOE_PROTOCOL_DISCOVERY,
325 .request_pl = &request_pl,
326 .request_pl_sz = sizeof(request_pl),
327 .response_pl = &response_pl,
328 .response_pl_sz = sizeof(response_pl),
329 .complete = pci_doe_task_complete,
330 .private = &c,
331 };
332 int rc;
333
334 rc = pci_doe_submit_task(doe_mb, &task);
335 if (rc < 0)
336 return rc;
337
338 wait_for_completion(&c);
339
340 if (task.rv != sizeof(response_pl))
341 return -EIO;
342
343 *vid = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_VID, response_pl);
344 *protocol = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL,
345 response_pl);
346 *index = FIELD_GET(PCI_DOE_DATA_OBJECT_DISC_RSP_3_NEXT_INDEX,
347 response_pl);
348
349 return 0;
350 }
351
pci_doe_xa_prot_entry(u16 vid,u8 prot)352 static void *pci_doe_xa_prot_entry(u16 vid, u8 prot)
353 {
354 return xa_mk_value((vid << 8) | prot);
355 }
356
pci_doe_cache_protocols(struct pci_doe_mb * doe_mb)357 static int pci_doe_cache_protocols(struct pci_doe_mb *doe_mb)
358 {
359 u8 index = 0;
360 u8 xa_idx = 0;
361
362 do {
363 int rc;
364 u16 vid;
365 u8 prot;
366
367 rc = pci_doe_discovery(doe_mb, &index, &vid, &prot);
368 if (rc)
369 return rc;
370
371 pci_dbg(doe_mb->pdev,
372 "[%x] Found protocol %d vid: %x prot: %x\n",
373 doe_mb->cap_offset, xa_idx, vid, prot);
374
375 rc = xa_insert(&doe_mb->prots, xa_idx++,
376 pci_doe_xa_prot_entry(vid, prot), GFP_KERNEL);
377 if (rc)
378 return rc;
379 } while (index);
380
381 return 0;
382 }
383
pci_doe_xa_destroy(void * mb)384 static void pci_doe_xa_destroy(void *mb)
385 {
386 struct pci_doe_mb *doe_mb = mb;
387
388 xa_destroy(&doe_mb->prots);
389 }
390
pci_doe_destroy_workqueue(void * mb)391 static void pci_doe_destroy_workqueue(void *mb)
392 {
393 struct pci_doe_mb *doe_mb = mb;
394
395 destroy_workqueue(doe_mb->work_queue);
396 }
397
pci_doe_flush_mb(void * mb)398 static void pci_doe_flush_mb(void *mb)
399 {
400 struct pci_doe_mb *doe_mb = mb;
401
402 /* Stop all pending work items from starting */
403 set_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags);
404
405 /* Cancel an in progress work item, if necessary */
406 set_bit(PCI_DOE_FLAG_CANCEL, &doe_mb->flags);
407 wake_up(&doe_mb->wq);
408
409 /* Flush all work items */
410 flush_workqueue(doe_mb->work_queue);
411 }
412
413 /**
414 * pcim_doe_create_mb() - Create a DOE mailbox object
415 *
416 * @pdev: PCI device to create the DOE mailbox for
417 * @cap_offset: Offset of the DOE mailbox
418 *
419 * Create a single mailbox object to manage the mailbox protocol at the
420 * cap_offset specified.
421 *
422 * RETURNS: created mailbox object on success
423 * ERR_PTR(-errno) on failure
424 */
pcim_doe_create_mb(struct pci_dev * pdev,u16 cap_offset)425 struct pci_doe_mb *pcim_doe_create_mb(struct pci_dev *pdev, u16 cap_offset)
426 {
427 struct pci_doe_mb *doe_mb;
428 struct device *dev = &pdev->dev;
429 int rc;
430
431 doe_mb = devm_kzalloc(dev, sizeof(*doe_mb), GFP_KERNEL);
432 if (!doe_mb)
433 return ERR_PTR(-ENOMEM);
434
435 doe_mb->pdev = pdev;
436 doe_mb->cap_offset = cap_offset;
437 init_waitqueue_head(&doe_mb->wq);
438
439 xa_init(&doe_mb->prots);
440 rc = devm_add_action(dev, pci_doe_xa_destroy, doe_mb);
441 if (rc)
442 return ERR_PTR(rc);
443
444 doe_mb->work_queue = alloc_ordered_workqueue("%s %s DOE [%x]", 0,
445 dev_driver_string(&pdev->dev),
446 pci_name(pdev),
447 doe_mb->cap_offset);
448 if (!doe_mb->work_queue) {
449 pci_err(pdev, "[%x] failed to allocate work queue\n",
450 doe_mb->cap_offset);
451 return ERR_PTR(-ENOMEM);
452 }
453 rc = devm_add_action_or_reset(dev, pci_doe_destroy_workqueue, doe_mb);
454 if (rc)
455 return ERR_PTR(rc);
456
457 /* Reset the mailbox by issuing an abort */
458 rc = pci_doe_abort(doe_mb);
459 if (rc) {
460 pci_err(pdev, "[%x] failed to reset mailbox with abort command : %d\n",
461 doe_mb->cap_offset, rc);
462 return ERR_PTR(rc);
463 }
464
465 /*
466 * The state machine and the mailbox should be in sync now;
467 * Set up mailbox flush prior to using the mailbox to query protocols.
468 */
469 rc = devm_add_action_or_reset(dev, pci_doe_flush_mb, doe_mb);
470 if (rc)
471 return ERR_PTR(rc);
472
473 rc = pci_doe_cache_protocols(doe_mb);
474 if (rc) {
475 pci_err(pdev, "[%x] failed to cache protocols : %d\n",
476 doe_mb->cap_offset, rc);
477 return ERR_PTR(rc);
478 }
479
480 return doe_mb;
481 }
482 EXPORT_SYMBOL_GPL(pcim_doe_create_mb);
483
484 /**
485 * pci_doe_supports_prot() - Return if the DOE instance supports the given
486 * protocol
487 * @doe_mb: DOE mailbox capability to query
488 * @vid: Protocol Vendor ID
489 * @type: Protocol type
490 *
491 * RETURNS: True if the DOE mailbox supports the protocol specified
492 */
pci_doe_supports_prot(struct pci_doe_mb * doe_mb,u16 vid,u8 type)493 bool pci_doe_supports_prot(struct pci_doe_mb *doe_mb, u16 vid, u8 type)
494 {
495 unsigned long index;
496 void *entry;
497
498 /* The discovery protocol must always be supported */
499 if (vid == PCI_VENDOR_ID_PCI_SIG && type == PCI_DOE_PROTOCOL_DISCOVERY)
500 return true;
501
502 xa_for_each(&doe_mb->prots, index, entry)
503 if (entry == pci_doe_xa_prot_entry(vid, type))
504 return true;
505
506 return false;
507 }
508 EXPORT_SYMBOL_GPL(pci_doe_supports_prot);
509
510 /**
511 * pci_doe_submit_task() - Submit a task to be processed by the state machine
512 *
513 * @doe_mb: DOE mailbox capability to submit to
514 * @task: task to be queued
515 *
516 * Submit a DOE task (request/response) to the DOE mailbox to be processed.
517 * Returns upon queueing the task object. If the queue is full this function
518 * will sleep until there is room in the queue.
519 *
520 * task->complete will be called when the state machine is done processing this
521 * task.
522 *
523 * Excess data will be discarded.
524 *
525 * RETURNS: 0 when task has been successfully queued, -ERRNO on error
526 */
pci_doe_submit_task(struct pci_doe_mb * doe_mb,struct pci_doe_task * task)527 int pci_doe_submit_task(struct pci_doe_mb *doe_mb, struct pci_doe_task *task)
528 {
529 if (!pci_doe_supports_prot(doe_mb, task->prot.vid, task->prot.type))
530 return -EINVAL;
531
532 /*
533 * DOE requests must be a whole number of DW and the response needs to
534 * be big enough for at least 1 DW
535 */
536 if (task->request_pl_sz % sizeof(u32) ||
537 task->response_pl_sz < sizeof(u32))
538 return -EINVAL;
539
540 if (test_bit(PCI_DOE_FLAG_DEAD, &doe_mb->flags))
541 return -EIO;
542
543 task->doe_mb = doe_mb;
544 INIT_WORK(&task->work, doe_statemachine_work);
545 queue_work(doe_mb->work_queue, &task->work);
546 return 0;
547 }
548 EXPORT_SYMBOL_GPL(pci_doe_submit_task);
549