1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2
3 #include <linux/module.h>
4 #include <linux/debugfs.h>
5 #include <linux/ntb.h>
6 #include <linux/pci.h>
7 #include <linux/radix-tree.h>
8 #include <linux/workqueue.h>
9
10 MODULE_LICENSE("Dual BSD/GPL");
11 MODULE_VERSION("0.1");
12 MODULE_AUTHOR("Logan Gunthorpe <logang@deltatee.com>");
13 MODULE_DESCRIPTION("Test for sending MSI interrupts over an NTB memory window");
14
15 static int num_irqs = 4;
16 module_param(num_irqs, int, 0644);
17 MODULE_PARM_DESC(num_irqs, "number of irqs to use");
18
19 struct ntb_msit_ctx {
20 struct ntb_dev *ntb;
21 struct dentry *dbgfs_dir;
22 struct work_struct setup_work;
23
24 struct ntb_msit_isr_ctx {
25 int irq_idx;
26 int irq_num;
27 int occurrences;
28 struct ntb_msit_ctx *nm;
29 struct ntb_msi_desc desc;
30 } *isr_ctx;
31
32 struct ntb_msit_peer {
33 struct ntb_msit_ctx *nm;
34 int pidx;
35 int num_irqs;
36 struct completion init_comp;
37 struct ntb_msi_desc *msi_desc;
38 } peers[];
39 };
40
41 static struct dentry *ntb_msit_dbgfs_topdir;
42
ntb_msit_isr(int irq,void * dev)43 static irqreturn_t ntb_msit_isr(int irq, void *dev)
44 {
45 struct ntb_msit_isr_ctx *isr_ctx = dev;
46 struct ntb_msit_ctx *nm = isr_ctx->nm;
47
48 dev_dbg(&nm->ntb->dev, "Interrupt Occurred: %d",
49 isr_ctx->irq_idx);
50
51 isr_ctx->occurrences++;
52
53 return IRQ_HANDLED;
54 }
55
ntb_msit_setup_work(struct work_struct * work)56 static void ntb_msit_setup_work(struct work_struct *work)
57 {
58 struct ntb_msit_ctx *nm = container_of(work, struct ntb_msit_ctx,
59 setup_work);
60 int irq_count = 0;
61 int irq;
62 int ret;
63 uintptr_t i;
64
65 ret = ntb_msi_setup_mws(nm->ntb);
66 if (ret) {
67 dev_err(&nm->ntb->dev, "Unable to setup MSI windows: %d\n",
68 ret);
69 return;
70 }
71
72 for (i = 0; i < num_irqs; i++) {
73 nm->isr_ctx[i].irq_idx = i;
74 nm->isr_ctx[i].nm = nm;
75
76 if (!nm->isr_ctx[i].irq_num) {
77 irq = ntbm_msi_request_irq(nm->ntb, ntb_msit_isr,
78 KBUILD_MODNAME,
79 &nm->isr_ctx[i],
80 &nm->isr_ctx[i].desc);
81 if (irq < 0)
82 break;
83
84 nm->isr_ctx[i].irq_num = irq;
85 }
86
87 ret = ntb_spad_write(nm->ntb, 2 * i + 1,
88 nm->isr_ctx[i].desc.addr_offset);
89 if (ret)
90 break;
91
92 ret = ntb_spad_write(nm->ntb, 2 * i + 2,
93 nm->isr_ctx[i].desc.data);
94 if (ret)
95 break;
96
97 irq_count++;
98 }
99
100 ntb_spad_write(nm->ntb, 0, irq_count);
101 ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb)));
102 }
103
ntb_msit_desc_changed(void * ctx)104 static void ntb_msit_desc_changed(void *ctx)
105 {
106 struct ntb_msit_ctx *nm = ctx;
107 int i;
108
109 dev_dbg(&nm->ntb->dev, "MSI Descriptors Changed\n");
110
111 for (i = 0; i < num_irqs; i++) {
112 ntb_spad_write(nm->ntb, 2 * i + 1,
113 nm->isr_ctx[i].desc.addr_offset);
114 ntb_spad_write(nm->ntb, 2 * i + 2,
115 nm->isr_ctx[i].desc.data);
116 }
117
118 ntb_peer_db_set(nm->ntb, BIT(ntb_port_number(nm->ntb)));
119 }
120
ntb_msit_link_event(void * ctx)121 static void ntb_msit_link_event(void *ctx)
122 {
123 struct ntb_msit_ctx *nm = ctx;
124
125 if (!ntb_link_is_up(nm->ntb, NULL, NULL))
126 return;
127
128 schedule_work(&nm->setup_work);
129 }
130
ntb_msit_copy_peer_desc(struct ntb_msit_ctx * nm,int peer)131 static void ntb_msit_copy_peer_desc(struct ntb_msit_ctx *nm, int peer)
132 {
133 int i;
134 struct ntb_msi_desc *desc = nm->peers[peer].msi_desc;
135 int irq_count = nm->peers[peer].num_irqs;
136
137 for (i = 0; i < irq_count; i++) {
138 desc[i].addr_offset = ntb_peer_spad_read(nm->ntb, peer,
139 2 * i + 1);
140 desc[i].data = ntb_peer_spad_read(nm->ntb, peer, 2 * i + 2);
141 }
142
143 dev_info(&nm->ntb->dev, "Found %d interrupts on peer %d\n",
144 irq_count, peer);
145
146 complete_all(&nm->peers[peer].init_comp);
147 }
148
ntb_msit_db_event(void * ctx,int vec)149 static void ntb_msit_db_event(void *ctx, int vec)
150 {
151 struct ntb_msit_ctx *nm = ctx;
152 struct ntb_msi_desc *desc;
153 u64 peer_mask = ntb_db_read(nm->ntb);
154 u32 irq_count;
155 int peer;
156
157 ntb_db_clear(nm->ntb, peer_mask);
158
159 for (peer = 0; peer < sizeof(peer_mask) * 8; peer++) {
160 if (!(peer_mask & BIT(peer)))
161 continue;
162
163 irq_count = ntb_peer_spad_read(nm->ntb, peer, 0);
164 if (irq_count == -1)
165 continue;
166
167 desc = kcalloc(irq_count, sizeof(*desc), GFP_ATOMIC);
168 if (!desc)
169 continue;
170
171 kfree(nm->peers[peer].msi_desc);
172 nm->peers[peer].msi_desc = desc;
173 nm->peers[peer].num_irqs = irq_count;
174
175 ntb_msit_copy_peer_desc(nm, peer);
176 }
177 }
178
179 static const struct ntb_ctx_ops ntb_msit_ops = {
180 .link_event = ntb_msit_link_event,
181 .db_event = ntb_msit_db_event,
182 };
183
ntb_msit_dbgfs_trigger(void * data,u64 idx)184 static int ntb_msit_dbgfs_trigger(void *data, u64 idx)
185 {
186 struct ntb_msit_peer *peer = data;
187
188 if (idx >= peer->num_irqs)
189 return -EINVAL;
190
191 dev_dbg(&peer->nm->ntb->dev, "trigger irq %llu on peer %u\n",
192 idx, peer->pidx);
193
194 return ntb_msi_peer_trigger(peer->nm->ntb, peer->pidx,
195 &peer->msi_desc[idx]);
196 }
197
198 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_trigger_fops, NULL,
199 ntb_msit_dbgfs_trigger, "%llu\n");
200
ntb_msit_dbgfs_port_get(void * data,u64 * port)201 static int ntb_msit_dbgfs_port_get(void *data, u64 *port)
202 {
203 struct ntb_msit_peer *peer = data;
204
205 *port = ntb_peer_port_number(peer->nm->ntb, peer->pidx);
206
207 return 0;
208 }
209
210 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_port_fops, ntb_msit_dbgfs_port_get,
211 NULL, "%llu\n");
212
ntb_msit_dbgfs_count_get(void * data,u64 * count)213 static int ntb_msit_dbgfs_count_get(void *data, u64 *count)
214 {
215 struct ntb_msit_peer *peer = data;
216
217 *count = peer->num_irqs;
218
219 return 0;
220 }
221
222 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_count_fops, ntb_msit_dbgfs_count_get,
223 NULL, "%llu\n");
224
ntb_msit_dbgfs_ready_get(void * data,u64 * ready)225 static int ntb_msit_dbgfs_ready_get(void *data, u64 *ready)
226 {
227 struct ntb_msit_peer *peer = data;
228
229 *ready = try_wait_for_completion(&peer->init_comp);
230
231 return 0;
232 }
233
ntb_msit_dbgfs_ready_set(void * data,u64 ready)234 static int ntb_msit_dbgfs_ready_set(void *data, u64 ready)
235 {
236 struct ntb_msit_peer *peer = data;
237
238 return wait_for_completion_interruptible(&peer->init_comp);
239 }
240
241 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_ready_fops, ntb_msit_dbgfs_ready_get,
242 ntb_msit_dbgfs_ready_set, "%llu\n");
243
ntb_msit_dbgfs_occurrences_get(void * data,u64 * occurrences)244 static int ntb_msit_dbgfs_occurrences_get(void *data, u64 *occurrences)
245 {
246 struct ntb_msit_isr_ctx *isr_ctx = data;
247
248 *occurrences = isr_ctx->occurrences;
249
250 return 0;
251 }
252
253 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_occurrences_fops,
254 ntb_msit_dbgfs_occurrences_get,
255 NULL, "%llu\n");
256
ntb_msit_dbgfs_local_port_get(void * data,u64 * port)257 static int ntb_msit_dbgfs_local_port_get(void *data, u64 *port)
258 {
259 struct ntb_msit_ctx *nm = data;
260
261 *port = ntb_port_number(nm->ntb);
262
263 return 0;
264 }
265
266 DEFINE_DEBUGFS_ATTRIBUTE(ntb_msit_local_port_fops,
267 ntb_msit_dbgfs_local_port_get,
268 NULL, "%llu\n");
269
ntb_msit_create_dbgfs(struct ntb_msit_ctx * nm)270 static void ntb_msit_create_dbgfs(struct ntb_msit_ctx *nm)
271 {
272 struct pci_dev *pdev = nm->ntb->pdev;
273 char buf[32];
274 int i;
275 struct dentry *peer_dir;
276
277 nm->dbgfs_dir = debugfs_create_dir(pci_name(pdev),
278 ntb_msit_dbgfs_topdir);
279 debugfs_create_file("port", 0400, nm->dbgfs_dir, nm,
280 &ntb_msit_local_port_fops);
281
282 for (i = 0; i < ntb_peer_port_count(nm->ntb); i++) {
283 nm->peers[i].pidx = i;
284 nm->peers[i].nm = nm;
285 init_completion(&nm->peers[i].init_comp);
286
287 snprintf(buf, sizeof(buf), "peer%d", i);
288 peer_dir = debugfs_create_dir(buf, nm->dbgfs_dir);
289
290 debugfs_create_file_unsafe("trigger", 0200, peer_dir,
291 &nm->peers[i],
292 &ntb_msit_trigger_fops);
293
294 debugfs_create_file_unsafe("port", 0400, peer_dir,
295 &nm->peers[i], &ntb_msit_port_fops);
296
297 debugfs_create_file_unsafe("count", 0400, peer_dir,
298 &nm->peers[i],
299 &ntb_msit_count_fops);
300
301 debugfs_create_file_unsafe("ready", 0600, peer_dir,
302 &nm->peers[i],
303 &ntb_msit_ready_fops);
304 }
305
306 for (i = 0; i < num_irqs; i++) {
307 snprintf(buf, sizeof(buf), "irq%d_occurrences", i);
308 debugfs_create_file_unsafe(buf, 0400, nm->dbgfs_dir,
309 &nm->isr_ctx[i],
310 &ntb_msit_occurrences_fops);
311 }
312 }
313
ntb_msit_remove_dbgfs(struct ntb_msit_ctx * nm)314 static void ntb_msit_remove_dbgfs(struct ntb_msit_ctx *nm)
315 {
316 debugfs_remove_recursive(nm->dbgfs_dir);
317 }
318
ntb_msit_probe(struct ntb_client * client,struct ntb_dev * ntb)319 static int ntb_msit_probe(struct ntb_client *client, struct ntb_dev *ntb)
320 {
321 struct ntb_msit_ctx *nm;
322 int peers;
323 int ret;
324
325 peers = ntb_peer_port_count(ntb);
326 if (peers <= 0)
327 return -EINVAL;
328
329 if (ntb_spad_is_unsafe(ntb) || ntb_spad_count(ntb) < 2 * num_irqs + 1) {
330 dev_err(&ntb->dev, "NTB MSI test requires at least %d spads for %d irqs\n",
331 2 * num_irqs + 1, num_irqs);
332 return -EFAULT;
333 }
334
335 ret = ntb_spad_write(ntb, 0, -1);
336 if (ret) {
337 dev_err(&ntb->dev, "Unable to write spads: %d\n", ret);
338 return ret;
339 }
340
341 ret = ntb_db_clear_mask(ntb, GENMASK(peers - 1, 0));
342 if (ret) {
343 dev_err(&ntb->dev, "Unable to clear doorbell mask: %d\n", ret);
344 return ret;
345 }
346
347 ret = ntb_msi_init(ntb, ntb_msit_desc_changed);
348 if (ret) {
349 dev_err(&ntb->dev, "Unable to initialize MSI library: %d\n",
350 ret);
351 return ret;
352 }
353
354 nm = devm_kzalloc(&ntb->dev, struct_size(nm, peers, peers), GFP_KERNEL);
355 if (!nm)
356 return -ENOMEM;
357
358 nm->isr_ctx = devm_kcalloc(&ntb->dev, num_irqs, sizeof(*nm->isr_ctx),
359 GFP_KERNEL);
360 if (!nm->isr_ctx)
361 return -ENOMEM;
362
363 INIT_WORK(&nm->setup_work, ntb_msit_setup_work);
364 nm->ntb = ntb;
365
366 ntb_msit_create_dbgfs(nm);
367
368 ret = ntb_set_ctx(ntb, nm, &ntb_msit_ops);
369 if (ret)
370 goto remove_dbgfs;
371
372 if (!nm->isr_ctx) {
373 ret = -ENOMEM;
374 goto remove_dbgfs;
375 }
376
377 ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
378
379 return 0;
380
381 remove_dbgfs:
382 ntb_msit_remove_dbgfs(nm);
383 devm_kfree(&ntb->dev, nm->isr_ctx);
384 devm_kfree(&ntb->dev, nm);
385 return ret;
386 }
387
ntb_msit_remove(struct ntb_client * client,struct ntb_dev * ntb)388 static void ntb_msit_remove(struct ntb_client *client, struct ntb_dev *ntb)
389 {
390 struct ntb_msit_ctx *nm = ntb->ctx;
391 int i;
392
393 ntb_link_disable(ntb);
394 ntb_db_set_mask(ntb, ntb_db_valid_mask(ntb));
395 ntb_msi_clear_mws(ntb);
396
397 for (i = 0; i < ntb_peer_port_count(ntb); i++)
398 kfree(nm->peers[i].msi_desc);
399
400 ntb_clear_ctx(ntb);
401 ntb_msit_remove_dbgfs(nm);
402 }
403
404 static struct ntb_client ntb_msit_client = {
405 .ops = {
406 .probe = ntb_msit_probe,
407 .remove = ntb_msit_remove
408 }
409 };
410
ntb_msit_init(void)411 static int __init ntb_msit_init(void)
412 {
413 int ret;
414
415 if (debugfs_initialized())
416 ntb_msit_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME,
417 NULL);
418
419 ret = ntb_register_client(&ntb_msit_client);
420 if (ret)
421 debugfs_remove_recursive(ntb_msit_dbgfs_topdir);
422
423 return ret;
424 }
425 module_init(ntb_msit_init);
426
ntb_msit_exit(void)427 static void __exit ntb_msit_exit(void)
428 {
429 ntb_unregister_client(&ntb_msit_client);
430 debugfs_remove_recursive(ntb_msit_dbgfs_topdir);
431 }
432 module_exit(ntb_msit_exit);
433