1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/device.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <uapi/linux/idxd.h>
10 #include "registers.h"
11 #include "idxd.h"
12
13 static char *idxd_wq_type_names[] = {
14 [IDXD_WQT_NONE] = "none",
15 [IDXD_WQT_KERNEL] = "kernel",
16 [IDXD_WQT_USER] = "user",
17 };
18
19 /* IDXD engine attributes */
engine_group_id_show(struct device * dev,struct device_attribute * attr,char * buf)20 static ssize_t engine_group_id_show(struct device *dev,
21 struct device_attribute *attr, char *buf)
22 {
23 struct idxd_engine *engine = confdev_to_engine(dev);
24
25 if (engine->group)
26 return sysfs_emit(buf, "%d\n", engine->group->id);
27 else
28 return sysfs_emit(buf, "%d\n", -1);
29 }
30
engine_group_id_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)31 static ssize_t engine_group_id_store(struct device *dev,
32 struct device_attribute *attr,
33 const char *buf, size_t count)
34 {
35 struct idxd_engine *engine = confdev_to_engine(dev);
36 struct idxd_device *idxd = engine->idxd;
37 long id;
38 int rc;
39 struct idxd_group *prevg;
40
41 rc = kstrtol(buf, 10, &id);
42 if (rc < 0)
43 return -EINVAL;
44
45 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
46 return -EPERM;
47
48 if (id > idxd->max_groups - 1 || id < -1)
49 return -EINVAL;
50
51 if (id == -1) {
52 if (engine->group) {
53 engine->group->num_engines--;
54 engine->group = NULL;
55 }
56 return count;
57 }
58
59 prevg = engine->group;
60
61 if (prevg)
62 prevg->num_engines--;
63 engine->group = idxd->groups[id];
64 engine->group->num_engines++;
65
66 return count;
67 }
68
69 static struct device_attribute dev_attr_engine_group =
70 __ATTR(group_id, 0644, engine_group_id_show,
71 engine_group_id_store);
72
73 static struct attribute *idxd_engine_attributes[] = {
74 &dev_attr_engine_group.attr,
75 NULL,
76 };
77
78 static const struct attribute_group idxd_engine_attribute_group = {
79 .attrs = idxd_engine_attributes,
80 };
81
82 static const struct attribute_group *idxd_engine_attribute_groups[] = {
83 &idxd_engine_attribute_group,
84 NULL,
85 };
86
idxd_conf_engine_release(struct device * dev)87 static void idxd_conf_engine_release(struct device *dev)
88 {
89 struct idxd_engine *engine = confdev_to_engine(dev);
90
91 kfree(engine);
92 }
93
94 struct device_type idxd_engine_device_type = {
95 .name = "engine",
96 .release = idxd_conf_engine_release,
97 .groups = idxd_engine_attribute_groups,
98 };
99
100 /* Group attributes */
101
idxd_set_free_rdbufs(struct idxd_device * idxd)102 static void idxd_set_free_rdbufs(struct idxd_device *idxd)
103 {
104 int i, rdbufs;
105
106 for (i = 0, rdbufs = 0; i < idxd->max_groups; i++) {
107 struct idxd_group *g = idxd->groups[i];
108
109 rdbufs += g->rdbufs_reserved;
110 }
111
112 idxd->nr_rdbufs = idxd->max_rdbufs - rdbufs;
113 }
114
group_read_buffers_reserved_show(struct device * dev,struct device_attribute * attr,char * buf)115 static ssize_t group_read_buffers_reserved_show(struct device *dev,
116 struct device_attribute *attr,
117 char *buf)
118 {
119 struct idxd_group *group = confdev_to_group(dev);
120
121 return sysfs_emit(buf, "%u\n", group->rdbufs_reserved);
122 }
123
group_tokens_reserved_show(struct device * dev,struct device_attribute * attr,char * buf)124 static ssize_t group_tokens_reserved_show(struct device *dev,
125 struct device_attribute *attr,
126 char *buf)
127 {
128 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n");
129 return group_read_buffers_reserved_show(dev, attr, buf);
130 }
131
group_read_buffers_reserved_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)132 static ssize_t group_read_buffers_reserved_store(struct device *dev,
133 struct device_attribute *attr,
134 const char *buf, size_t count)
135 {
136 struct idxd_group *group = confdev_to_group(dev);
137 struct idxd_device *idxd = group->idxd;
138 unsigned long val;
139 int rc;
140
141 rc = kstrtoul(buf, 10, &val);
142 if (rc < 0)
143 return -EINVAL;
144
145 if (idxd->data->type == IDXD_TYPE_IAX)
146 return -EOPNOTSUPP;
147
148 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
149 return -EPERM;
150
151 if (idxd->state == IDXD_DEV_ENABLED)
152 return -EPERM;
153
154 if (val > idxd->max_rdbufs)
155 return -EINVAL;
156
157 if (val > idxd->nr_rdbufs + group->rdbufs_reserved)
158 return -EINVAL;
159
160 group->rdbufs_reserved = val;
161 idxd_set_free_rdbufs(idxd);
162 return count;
163 }
164
group_tokens_reserved_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)165 static ssize_t group_tokens_reserved_store(struct device *dev,
166 struct device_attribute *attr,
167 const char *buf, size_t count)
168 {
169 dev_warn_once(dev, "attribute deprecated, see read_buffers_reserved.\n");
170 return group_read_buffers_reserved_store(dev, attr, buf, count);
171 }
172
173 static struct device_attribute dev_attr_group_tokens_reserved =
174 __ATTR(tokens_reserved, 0644, group_tokens_reserved_show,
175 group_tokens_reserved_store);
176
177 static struct device_attribute dev_attr_group_read_buffers_reserved =
178 __ATTR(read_buffers_reserved, 0644, group_read_buffers_reserved_show,
179 group_read_buffers_reserved_store);
180
group_read_buffers_allowed_show(struct device * dev,struct device_attribute * attr,char * buf)181 static ssize_t group_read_buffers_allowed_show(struct device *dev,
182 struct device_attribute *attr,
183 char *buf)
184 {
185 struct idxd_group *group = confdev_to_group(dev);
186
187 return sysfs_emit(buf, "%u\n", group->rdbufs_allowed);
188 }
189
group_tokens_allowed_show(struct device * dev,struct device_attribute * attr,char * buf)190 static ssize_t group_tokens_allowed_show(struct device *dev,
191 struct device_attribute *attr,
192 char *buf)
193 {
194 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n");
195 return group_read_buffers_allowed_show(dev, attr, buf);
196 }
197
group_read_buffers_allowed_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)198 static ssize_t group_read_buffers_allowed_store(struct device *dev,
199 struct device_attribute *attr,
200 const char *buf, size_t count)
201 {
202 struct idxd_group *group = confdev_to_group(dev);
203 struct idxd_device *idxd = group->idxd;
204 unsigned long val;
205 int rc;
206
207 rc = kstrtoul(buf, 10, &val);
208 if (rc < 0)
209 return -EINVAL;
210
211 if (idxd->data->type == IDXD_TYPE_IAX)
212 return -EOPNOTSUPP;
213
214 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
215 return -EPERM;
216
217 if (idxd->state == IDXD_DEV_ENABLED)
218 return -EPERM;
219
220 if (val < 4 * group->num_engines ||
221 val > group->rdbufs_reserved + idxd->nr_rdbufs)
222 return -EINVAL;
223
224 group->rdbufs_allowed = val;
225 return count;
226 }
227
group_tokens_allowed_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)228 static ssize_t group_tokens_allowed_store(struct device *dev,
229 struct device_attribute *attr,
230 const char *buf, size_t count)
231 {
232 dev_warn_once(dev, "attribute deprecated, see read_buffers_allowed.\n");
233 return group_read_buffers_allowed_store(dev, attr, buf, count);
234 }
235
236 static struct device_attribute dev_attr_group_tokens_allowed =
237 __ATTR(tokens_allowed, 0644, group_tokens_allowed_show,
238 group_tokens_allowed_store);
239
240 static struct device_attribute dev_attr_group_read_buffers_allowed =
241 __ATTR(read_buffers_allowed, 0644, group_read_buffers_allowed_show,
242 group_read_buffers_allowed_store);
243
group_use_read_buffer_limit_show(struct device * dev,struct device_attribute * attr,char * buf)244 static ssize_t group_use_read_buffer_limit_show(struct device *dev,
245 struct device_attribute *attr,
246 char *buf)
247 {
248 struct idxd_group *group = confdev_to_group(dev);
249
250 return sysfs_emit(buf, "%u\n", group->use_rdbuf_limit);
251 }
252
group_use_token_limit_show(struct device * dev,struct device_attribute * attr,char * buf)253 static ssize_t group_use_token_limit_show(struct device *dev,
254 struct device_attribute *attr,
255 char *buf)
256 {
257 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n");
258 return group_use_read_buffer_limit_show(dev, attr, buf);
259 }
260
group_use_read_buffer_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)261 static ssize_t group_use_read_buffer_limit_store(struct device *dev,
262 struct device_attribute *attr,
263 const char *buf, size_t count)
264 {
265 struct idxd_group *group = confdev_to_group(dev);
266 struct idxd_device *idxd = group->idxd;
267 unsigned long val;
268 int rc;
269
270 rc = kstrtoul(buf, 10, &val);
271 if (rc < 0)
272 return -EINVAL;
273
274 if (idxd->data->type == IDXD_TYPE_IAX)
275 return -EOPNOTSUPP;
276
277 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
278 return -EPERM;
279
280 if (idxd->state == IDXD_DEV_ENABLED)
281 return -EPERM;
282
283 if (idxd->rdbuf_limit == 0)
284 return -EPERM;
285
286 group->use_rdbuf_limit = !!val;
287 return count;
288 }
289
group_use_token_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)290 static ssize_t group_use_token_limit_store(struct device *dev,
291 struct device_attribute *attr,
292 const char *buf, size_t count)
293 {
294 dev_warn_once(dev, "attribute deprecated, see use_read_buffer_limit.\n");
295 return group_use_read_buffer_limit_store(dev, attr, buf, count);
296 }
297
298 static struct device_attribute dev_attr_group_use_token_limit =
299 __ATTR(use_token_limit, 0644, group_use_token_limit_show,
300 group_use_token_limit_store);
301
302 static struct device_attribute dev_attr_group_use_read_buffer_limit =
303 __ATTR(use_read_buffer_limit, 0644, group_use_read_buffer_limit_show,
304 group_use_read_buffer_limit_store);
305
group_engines_show(struct device * dev,struct device_attribute * attr,char * buf)306 static ssize_t group_engines_show(struct device *dev,
307 struct device_attribute *attr, char *buf)
308 {
309 struct idxd_group *group = confdev_to_group(dev);
310 int i, rc = 0;
311 struct idxd_device *idxd = group->idxd;
312
313 for (i = 0; i < idxd->max_engines; i++) {
314 struct idxd_engine *engine = idxd->engines[i];
315
316 if (!engine->group)
317 continue;
318
319 if (engine->group->id == group->id)
320 rc += sysfs_emit_at(buf, rc, "engine%d.%d ", idxd->id, engine->id);
321 }
322
323 if (!rc)
324 return 0;
325 rc--;
326 rc += sysfs_emit_at(buf, rc, "\n");
327
328 return rc;
329 }
330
331 static struct device_attribute dev_attr_group_engines =
332 __ATTR(engines, 0444, group_engines_show, NULL);
333
group_work_queues_show(struct device * dev,struct device_attribute * attr,char * buf)334 static ssize_t group_work_queues_show(struct device *dev,
335 struct device_attribute *attr, char *buf)
336 {
337 struct idxd_group *group = confdev_to_group(dev);
338 int i, rc = 0;
339 struct idxd_device *idxd = group->idxd;
340
341 for (i = 0; i < idxd->max_wqs; i++) {
342 struct idxd_wq *wq = idxd->wqs[i];
343
344 if (!wq->group)
345 continue;
346
347 if (wq->group->id == group->id)
348 rc += sysfs_emit_at(buf, rc, "wq%d.%d ", idxd->id, wq->id);
349 }
350
351 if (!rc)
352 return 0;
353 rc--;
354 rc += sysfs_emit_at(buf, rc, "\n");
355
356 return rc;
357 }
358
359 static struct device_attribute dev_attr_group_work_queues =
360 __ATTR(work_queues, 0444, group_work_queues_show, NULL);
361
group_traffic_class_a_show(struct device * dev,struct device_attribute * attr,char * buf)362 static ssize_t group_traffic_class_a_show(struct device *dev,
363 struct device_attribute *attr,
364 char *buf)
365 {
366 struct idxd_group *group = confdev_to_group(dev);
367
368 return sysfs_emit(buf, "%d\n", group->tc_a);
369 }
370
group_traffic_class_a_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)371 static ssize_t group_traffic_class_a_store(struct device *dev,
372 struct device_attribute *attr,
373 const char *buf, size_t count)
374 {
375 struct idxd_group *group = confdev_to_group(dev);
376 struct idxd_device *idxd = group->idxd;
377 long val;
378 int rc;
379
380 rc = kstrtol(buf, 10, &val);
381 if (rc < 0)
382 return -EINVAL;
383
384 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
385 return -EPERM;
386
387 if (idxd->state == IDXD_DEV_ENABLED)
388 return -EPERM;
389
390 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override)
391 return -EPERM;
392
393 if (val < 0 || val > 7)
394 return -EINVAL;
395
396 group->tc_a = val;
397 return count;
398 }
399
400 static struct device_attribute dev_attr_group_traffic_class_a =
401 __ATTR(traffic_class_a, 0644, group_traffic_class_a_show,
402 group_traffic_class_a_store);
403
group_traffic_class_b_show(struct device * dev,struct device_attribute * attr,char * buf)404 static ssize_t group_traffic_class_b_show(struct device *dev,
405 struct device_attribute *attr,
406 char *buf)
407 {
408 struct idxd_group *group = confdev_to_group(dev);
409
410 return sysfs_emit(buf, "%d\n", group->tc_b);
411 }
412
group_traffic_class_b_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)413 static ssize_t group_traffic_class_b_store(struct device *dev,
414 struct device_attribute *attr,
415 const char *buf, size_t count)
416 {
417 struct idxd_group *group = confdev_to_group(dev);
418 struct idxd_device *idxd = group->idxd;
419 long val;
420 int rc;
421
422 rc = kstrtol(buf, 10, &val);
423 if (rc < 0)
424 return -EINVAL;
425
426 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
427 return -EPERM;
428
429 if (idxd->state == IDXD_DEV_ENABLED)
430 return -EPERM;
431
432 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override)
433 return -EPERM;
434
435 if (val < 0 || val > 7)
436 return -EINVAL;
437
438 group->tc_b = val;
439 return count;
440 }
441
442 static struct device_attribute dev_attr_group_traffic_class_b =
443 __ATTR(traffic_class_b, 0644, group_traffic_class_b_show,
444 group_traffic_class_b_store);
445
group_desc_progress_limit_show(struct device * dev,struct device_attribute * attr,char * buf)446 static ssize_t group_desc_progress_limit_show(struct device *dev,
447 struct device_attribute *attr,
448 char *buf)
449 {
450 struct idxd_group *group = confdev_to_group(dev);
451
452 return sysfs_emit(buf, "%d\n", group->desc_progress_limit);
453 }
454
group_desc_progress_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)455 static ssize_t group_desc_progress_limit_store(struct device *dev,
456 struct device_attribute *attr,
457 const char *buf, size_t count)
458 {
459 struct idxd_group *group = confdev_to_group(dev);
460 int val, rc;
461
462 rc = kstrtoint(buf, 10, &val);
463 if (rc < 0)
464 return -EINVAL;
465
466 if (val & ~GENMASK(1, 0))
467 return -EINVAL;
468
469 group->desc_progress_limit = val;
470 return count;
471 }
472
473 static struct device_attribute dev_attr_group_desc_progress_limit =
474 __ATTR(desc_progress_limit, 0644, group_desc_progress_limit_show,
475 group_desc_progress_limit_store);
476
group_batch_progress_limit_show(struct device * dev,struct device_attribute * attr,char * buf)477 static ssize_t group_batch_progress_limit_show(struct device *dev,
478 struct device_attribute *attr,
479 char *buf)
480 {
481 struct idxd_group *group = confdev_to_group(dev);
482
483 return sysfs_emit(buf, "%d\n", group->batch_progress_limit);
484 }
485
group_batch_progress_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)486 static ssize_t group_batch_progress_limit_store(struct device *dev,
487 struct device_attribute *attr,
488 const char *buf, size_t count)
489 {
490 struct idxd_group *group = confdev_to_group(dev);
491 int val, rc;
492
493 rc = kstrtoint(buf, 10, &val);
494 if (rc < 0)
495 return -EINVAL;
496
497 if (val & ~GENMASK(1, 0))
498 return -EINVAL;
499
500 group->batch_progress_limit = val;
501 return count;
502 }
503
504 static struct device_attribute dev_attr_group_batch_progress_limit =
505 __ATTR(batch_progress_limit, 0644, group_batch_progress_limit_show,
506 group_batch_progress_limit_store);
507 static struct attribute *idxd_group_attributes[] = {
508 &dev_attr_group_work_queues.attr,
509 &dev_attr_group_engines.attr,
510 &dev_attr_group_use_token_limit.attr,
511 &dev_attr_group_use_read_buffer_limit.attr,
512 &dev_attr_group_tokens_allowed.attr,
513 &dev_attr_group_read_buffers_allowed.attr,
514 &dev_attr_group_tokens_reserved.attr,
515 &dev_attr_group_read_buffers_reserved.attr,
516 &dev_attr_group_traffic_class_a.attr,
517 &dev_attr_group_traffic_class_b.attr,
518 &dev_attr_group_desc_progress_limit.attr,
519 &dev_attr_group_batch_progress_limit.attr,
520 NULL,
521 };
522
idxd_group_attr_progress_limit_invisible(struct attribute * attr,struct idxd_device * idxd)523 static bool idxd_group_attr_progress_limit_invisible(struct attribute *attr,
524 struct idxd_device *idxd)
525 {
526 return (attr == &dev_attr_group_desc_progress_limit.attr ||
527 attr == &dev_attr_group_batch_progress_limit.attr) &&
528 !idxd->hw.group_cap.progress_limit;
529 }
530
idxd_group_attr_read_buffers_invisible(struct attribute * attr,struct idxd_device * idxd)531 static bool idxd_group_attr_read_buffers_invisible(struct attribute *attr,
532 struct idxd_device *idxd)
533 {
534 /*
535 * Intel IAA does not support Read Buffer allocation control,
536 * make these attributes invisible.
537 */
538 return (attr == &dev_attr_group_use_token_limit.attr ||
539 attr == &dev_attr_group_use_read_buffer_limit.attr ||
540 attr == &dev_attr_group_tokens_allowed.attr ||
541 attr == &dev_attr_group_read_buffers_allowed.attr ||
542 attr == &dev_attr_group_tokens_reserved.attr ||
543 attr == &dev_attr_group_read_buffers_reserved.attr) &&
544 idxd->data->type == IDXD_TYPE_IAX;
545 }
546
idxd_group_attr_visible(struct kobject * kobj,struct attribute * attr,int n)547 static umode_t idxd_group_attr_visible(struct kobject *kobj,
548 struct attribute *attr, int n)
549 {
550 struct device *dev = container_of(kobj, struct device, kobj);
551 struct idxd_group *group = confdev_to_group(dev);
552 struct idxd_device *idxd = group->idxd;
553
554 if (idxd_group_attr_progress_limit_invisible(attr, idxd))
555 return 0;
556
557 if (idxd_group_attr_read_buffers_invisible(attr, idxd))
558 return 0;
559
560 return attr->mode;
561 }
562
563 static const struct attribute_group idxd_group_attribute_group = {
564 .attrs = idxd_group_attributes,
565 .is_visible = idxd_group_attr_visible,
566 };
567
568 static const struct attribute_group *idxd_group_attribute_groups[] = {
569 &idxd_group_attribute_group,
570 NULL,
571 };
572
idxd_conf_group_release(struct device * dev)573 static void idxd_conf_group_release(struct device *dev)
574 {
575 struct idxd_group *group = confdev_to_group(dev);
576
577 kfree(group);
578 }
579
580 struct device_type idxd_group_device_type = {
581 .name = "group",
582 .release = idxd_conf_group_release,
583 .groups = idxd_group_attribute_groups,
584 };
585
586 /* IDXD work queue attribs */
wq_clients_show(struct device * dev,struct device_attribute * attr,char * buf)587 static ssize_t wq_clients_show(struct device *dev,
588 struct device_attribute *attr, char *buf)
589 {
590 struct idxd_wq *wq = confdev_to_wq(dev);
591
592 return sysfs_emit(buf, "%d\n", wq->client_count);
593 }
594
595 static struct device_attribute dev_attr_wq_clients =
596 __ATTR(clients, 0444, wq_clients_show, NULL);
597
wq_state_show(struct device * dev,struct device_attribute * attr,char * buf)598 static ssize_t wq_state_show(struct device *dev,
599 struct device_attribute *attr, char *buf)
600 {
601 struct idxd_wq *wq = confdev_to_wq(dev);
602
603 switch (wq->state) {
604 case IDXD_WQ_DISABLED:
605 return sysfs_emit(buf, "disabled\n");
606 case IDXD_WQ_ENABLED:
607 return sysfs_emit(buf, "enabled\n");
608 }
609
610 return sysfs_emit(buf, "unknown\n");
611 }
612
613 static struct device_attribute dev_attr_wq_state =
614 __ATTR(state, 0444, wq_state_show, NULL);
615
wq_group_id_show(struct device * dev,struct device_attribute * attr,char * buf)616 static ssize_t wq_group_id_show(struct device *dev,
617 struct device_attribute *attr, char *buf)
618 {
619 struct idxd_wq *wq = confdev_to_wq(dev);
620
621 if (wq->group)
622 return sysfs_emit(buf, "%u\n", wq->group->id);
623 else
624 return sysfs_emit(buf, "-1\n");
625 }
626
wq_group_id_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)627 static ssize_t wq_group_id_store(struct device *dev,
628 struct device_attribute *attr,
629 const char *buf, size_t count)
630 {
631 struct idxd_wq *wq = confdev_to_wq(dev);
632 struct idxd_device *idxd = wq->idxd;
633 long id;
634 int rc;
635 struct idxd_group *prevg, *group;
636
637 rc = kstrtol(buf, 10, &id);
638 if (rc < 0)
639 return -EINVAL;
640
641 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
642 return -EPERM;
643
644 if (wq->state != IDXD_WQ_DISABLED)
645 return -EPERM;
646
647 if (id > idxd->max_groups - 1 || id < -1)
648 return -EINVAL;
649
650 if (id == -1) {
651 if (wq->group) {
652 wq->group->num_wqs--;
653 wq->group = NULL;
654 }
655 return count;
656 }
657
658 group = idxd->groups[id];
659 prevg = wq->group;
660
661 if (prevg)
662 prevg->num_wqs--;
663 wq->group = group;
664 group->num_wqs++;
665 return count;
666 }
667
668 static struct device_attribute dev_attr_wq_group_id =
669 __ATTR(group_id, 0644, wq_group_id_show, wq_group_id_store);
670
wq_mode_show(struct device * dev,struct device_attribute * attr,char * buf)671 static ssize_t wq_mode_show(struct device *dev, struct device_attribute *attr,
672 char *buf)
673 {
674 struct idxd_wq *wq = confdev_to_wq(dev);
675
676 return sysfs_emit(buf, "%s\n", wq_dedicated(wq) ? "dedicated" : "shared");
677 }
678
wq_mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)679 static ssize_t wq_mode_store(struct device *dev,
680 struct device_attribute *attr, const char *buf,
681 size_t count)
682 {
683 struct idxd_wq *wq = confdev_to_wq(dev);
684 struct idxd_device *idxd = wq->idxd;
685
686 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
687 return -EPERM;
688
689 if (wq->state != IDXD_WQ_DISABLED)
690 return -EPERM;
691
692 if (sysfs_streq(buf, "dedicated")) {
693 set_bit(WQ_FLAG_DEDICATED, &wq->flags);
694 wq->threshold = 0;
695 } else if (sysfs_streq(buf, "shared")) {
696 clear_bit(WQ_FLAG_DEDICATED, &wq->flags);
697 } else {
698 return -EINVAL;
699 }
700
701 return count;
702 }
703
704 static struct device_attribute dev_attr_wq_mode =
705 __ATTR(mode, 0644, wq_mode_show, wq_mode_store);
706
wq_size_show(struct device * dev,struct device_attribute * attr,char * buf)707 static ssize_t wq_size_show(struct device *dev, struct device_attribute *attr,
708 char *buf)
709 {
710 struct idxd_wq *wq = confdev_to_wq(dev);
711
712 return sysfs_emit(buf, "%u\n", wq->size);
713 }
714
total_claimed_wq_size(struct idxd_device * idxd)715 static int total_claimed_wq_size(struct idxd_device *idxd)
716 {
717 int i;
718 int wq_size = 0;
719
720 for (i = 0; i < idxd->max_wqs; i++) {
721 struct idxd_wq *wq = idxd->wqs[i];
722
723 wq_size += wq->size;
724 }
725
726 return wq_size;
727 }
728
wq_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)729 static ssize_t wq_size_store(struct device *dev,
730 struct device_attribute *attr, const char *buf,
731 size_t count)
732 {
733 struct idxd_wq *wq = confdev_to_wq(dev);
734 unsigned long size;
735 struct idxd_device *idxd = wq->idxd;
736 int rc;
737
738 rc = kstrtoul(buf, 10, &size);
739 if (rc < 0)
740 return -EINVAL;
741
742 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
743 return -EPERM;
744
745 if (idxd->state == IDXD_DEV_ENABLED)
746 return -EPERM;
747
748 if (size + total_claimed_wq_size(idxd) - wq->size > idxd->max_wq_size)
749 return -EINVAL;
750
751 wq->size = size;
752 return count;
753 }
754
755 static struct device_attribute dev_attr_wq_size =
756 __ATTR(size, 0644, wq_size_show, wq_size_store);
757
wq_priority_show(struct device * dev,struct device_attribute * attr,char * buf)758 static ssize_t wq_priority_show(struct device *dev,
759 struct device_attribute *attr, char *buf)
760 {
761 struct idxd_wq *wq = confdev_to_wq(dev);
762
763 return sysfs_emit(buf, "%u\n", wq->priority);
764 }
765
wq_priority_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)766 static ssize_t wq_priority_store(struct device *dev,
767 struct device_attribute *attr,
768 const char *buf, size_t count)
769 {
770 struct idxd_wq *wq = confdev_to_wq(dev);
771 unsigned long prio;
772 struct idxd_device *idxd = wq->idxd;
773 int rc;
774
775 rc = kstrtoul(buf, 10, &prio);
776 if (rc < 0)
777 return -EINVAL;
778
779 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
780 return -EPERM;
781
782 if (wq->state != IDXD_WQ_DISABLED)
783 return -EPERM;
784
785 if (prio > IDXD_MAX_PRIORITY)
786 return -EINVAL;
787
788 wq->priority = prio;
789 return count;
790 }
791
792 static struct device_attribute dev_attr_wq_priority =
793 __ATTR(priority, 0644, wq_priority_show, wq_priority_store);
794
wq_block_on_fault_show(struct device * dev,struct device_attribute * attr,char * buf)795 static ssize_t wq_block_on_fault_show(struct device *dev,
796 struct device_attribute *attr, char *buf)
797 {
798 struct idxd_wq *wq = confdev_to_wq(dev);
799
800 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags));
801 }
802
wq_block_on_fault_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)803 static ssize_t wq_block_on_fault_store(struct device *dev,
804 struct device_attribute *attr,
805 const char *buf, size_t count)
806 {
807 struct idxd_wq *wq = confdev_to_wq(dev);
808 struct idxd_device *idxd = wq->idxd;
809 bool bof;
810 int rc;
811
812 if (!idxd->hw.gen_cap.block_on_fault)
813 return -EOPNOTSUPP;
814
815 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
816 return -EPERM;
817
818 if (wq->state != IDXD_WQ_DISABLED)
819 return -ENXIO;
820
821 rc = kstrtobool(buf, &bof);
822 if (rc < 0)
823 return rc;
824
825 if (bof)
826 set_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
827 else
828 clear_bit(WQ_FLAG_BLOCK_ON_FAULT, &wq->flags);
829
830 return count;
831 }
832
833 static struct device_attribute dev_attr_wq_block_on_fault =
834 __ATTR(block_on_fault, 0644, wq_block_on_fault_show,
835 wq_block_on_fault_store);
836
wq_threshold_show(struct device * dev,struct device_attribute * attr,char * buf)837 static ssize_t wq_threshold_show(struct device *dev,
838 struct device_attribute *attr, char *buf)
839 {
840 struct idxd_wq *wq = confdev_to_wq(dev);
841
842 return sysfs_emit(buf, "%u\n", wq->threshold);
843 }
844
wq_threshold_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)845 static ssize_t wq_threshold_store(struct device *dev,
846 struct device_attribute *attr,
847 const char *buf, size_t count)
848 {
849 struct idxd_wq *wq = confdev_to_wq(dev);
850 struct idxd_device *idxd = wq->idxd;
851 unsigned int val;
852 int rc;
853
854 rc = kstrtouint(buf, 0, &val);
855 if (rc < 0)
856 return -EINVAL;
857
858 if (val > wq->size || val <= 0)
859 return -EINVAL;
860
861 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
862 return -EPERM;
863
864 if (wq->state != IDXD_WQ_DISABLED)
865 return -ENXIO;
866
867 if (test_bit(WQ_FLAG_DEDICATED, &wq->flags))
868 return -EINVAL;
869
870 wq->threshold = val;
871
872 return count;
873 }
874
875 static struct device_attribute dev_attr_wq_threshold =
876 __ATTR(threshold, 0644, wq_threshold_show, wq_threshold_store);
877
wq_type_show(struct device * dev,struct device_attribute * attr,char * buf)878 static ssize_t wq_type_show(struct device *dev,
879 struct device_attribute *attr, char *buf)
880 {
881 struct idxd_wq *wq = confdev_to_wq(dev);
882
883 switch (wq->type) {
884 case IDXD_WQT_KERNEL:
885 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_KERNEL]);
886 case IDXD_WQT_USER:
887 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_USER]);
888 case IDXD_WQT_NONE:
889 default:
890 return sysfs_emit(buf, "%s\n", idxd_wq_type_names[IDXD_WQT_NONE]);
891 }
892
893 return -EINVAL;
894 }
895
wq_type_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)896 static ssize_t wq_type_store(struct device *dev,
897 struct device_attribute *attr, const char *buf,
898 size_t count)
899 {
900 struct idxd_wq *wq = confdev_to_wq(dev);
901 enum idxd_wq_type old_type;
902
903 if (wq->state != IDXD_WQ_DISABLED)
904 return -EPERM;
905
906 old_type = wq->type;
907 if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_NONE]))
908 wq->type = IDXD_WQT_NONE;
909 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_KERNEL]))
910 wq->type = IDXD_WQT_KERNEL;
911 else if (sysfs_streq(buf, idxd_wq_type_names[IDXD_WQT_USER]))
912 wq->type = IDXD_WQT_USER;
913 else
914 return -EINVAL;
915
916 /* If we are changing queue type, clear the name */
917 if (wq->type != old_type)
918 memset(wq->name, 0, WQ_NAME_SIZE + 1);
919
920 return count;
921 }
922
923 static struct device_attribute dev_attr_wq_type =
924 __ATTR(type, 0644, wq_type_show, wq_type_store);
925
wq_name_show(struct device * dev,struct device_attribute * attr,char * buf)926 static ssize_t wq_name_show(struct device *dev,
927 struct device_attribute *attr, char *buf)
928 {
929 struct idxd_wq *wq = confdev_to_wq(dev);
930
931 return sysfs_emit(buf, "%s\n", wq->name);
932 }
933
wq_name_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)934 static ssize_t wq_name_store(struct device *dev,
935 struct device_attribute *attr, const char *buf,
936 size_t count)
937 {
938 struct idxd_wq *wq = confdev_to_wq(dev);
939 char *input, *pos;
940
941 if (wq->state != IDXD_WQ_DISABLED)
942 return -EPERM;
943
944 if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)
945 return -EINVAL;
946
947 /*
948 * This is temporarily placed here until we have SVM support for
949 * dmaengine.
950 */
951 if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
952 return -EOPNOTSUPP;
953
954 input = kstrndup(buf, count, GFP_KERNEL);
955 if (!input)
956 return -ENOMEM;
957
958 pos = strim(input);
959 memset(wq->name, 0, WQ_NAME_SIZE + 1);
960 sprintf(wq->name, "%s", pos);
961 kfree(input);
962 return count;
963 }
964
965 static struct device_attribute dev_attr_wq_name =
966 __ATTR(name, 0644, wq_name_show, wq_name_store);
967
wq_cdev_minor_show(struct device * dev,struct device_attribute * attr,char * buf)968 static ssize_t wq_cdev_minor_show(struct device *dev,
969 struct device_attribute *attr, char *buf)
970 {
971 struct idxd_wq *wq = confdev_to_wq(dev);
972 int minor = -1;
973
974 mutex_lock(&wq->wq_lock);
975 if (wq->idxd_cdev)
976 minor = wq->idxd_cdev->minor;
977 mutex_unlock(&wq->wq_lock);
978
979 if (minor == -1)
980 return -ENXIO;
981 return sysfs_emit(buf, "%d\n", minor);
982 }
983
984 static struct device_attribute dev_attr_wq_cdev_minor =
985 __ATTR(cdev_minor, 0444, wq_cdev_minor_show, NULL);
986
__get_sysfs_u64(const char * buf,u64 * val)987 static int __get_sysfs_u64(const char *buf, u64 *val)
988 {
989 int rc;
990
991 rc = kstrtou64(buf, 0, val);
992 if (rc < 0)
993 return -EINVAL;
994
995 if (*val == 0)
996 return -EINVAL;
997
998 *val = roundup_pow_of_two(*val);
999 return 0;
1000 }
1001
wq_max_transfer_size_show(struct device * dev,struct device_attribute * attr,char * buf)1002 static ssize_t wq_max_transfer_size_show(struct device *dev, struct device_attribute *attr,
1003 char *buf)
1004 {
1005 struct idxd_wq *wq = confdev_to_wq(dev);
1006
1007 return sysfs_emit(buf, "%llu\n", wq->max_xfer_bytes);
1008 }
1009
wq_max_transfer_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1010 static ssize_t wq_max_transfer_size_store(struct device *dev, struct device_attribute *attr,
1011 const char *buf, size_t count)
1012 {
1013 struct idxd_wq *wq = confdev_to_wq(dev);
1014 struct idxd_device *idxd = wq->idxd;
1015 u64 xfer_size;
1016 int rc;
1017
1018 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1019 return -EPERM;
1020
1021 if (wq->state != IDXD_WQ_DISABLED)
1022 return -EPERM;
1023
1024 rc = __get_sysfs_u64(buf, &xfer_size);
1025 if (rc < 0)
1026 return rc;
1027
1028 if (xfer_size > idxd->max_xfer_bytes)
1029 return -EINVAL;
1030
1031 wq->max_xfer_bytes = xfer_size;
1032
1033 return count;
1034 }
1035
1036 static struct device_attribute dev_attr_wq_max_transfer_size =
1037 __ATTR(max_transfer_size, 0644,
1038 wq_max_transfer_size_show, wq_max_transfer_size_store);
1039
wq_max_batch_size_show(struct device * dev,struct device_attribute * attr,char * buf)1040 static ssize_t wq_max_batch_size_show(struct device *dev, struct device_attribute *attr, char *buf)
1041 {
1042 struct idxd_wq *wq = confdev_to_wq(dev);
1043
1044 return sysfs_emit(buf, "%u\n", wq->max_batch_size);
1045 }
1046
wq_max_batch_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1047 static ssize_t wq_max_batch_size_store(struct device *dev, struct device_attribute *attr,
1048 const char *buf, size_t count)
1049 {
1050 struct idxd_wq *wq = confdev_to_wq(dev);
1051 struct idxd_device *idxd = wq->idxd;
1052 u64 batch_size;
1053 int rc;
1054
1055 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1056 return -EPERM;
1057
1058 if (wq->state != IDXD_WQ_DISABLED)
1059 return -EPERM;
1060
1061 rc = __get_sysfs_u64(buf, &batch_size);
1062 if (rc < 0)
1063 return rc;
1064
1065 if (batch_size > idxd->max_batch_size)
1066 return -EINVAL;
1067
1068 idxd_wq_set_max_batch_size(idxd->data->type, wq, (u32)batch_size);
1069
1070 return count;
1071 }
1072
1073 static struct device_attribute dev_attr_wq_max_batch_size =
1074 __ATTR(max_batch_size, 0644, wq_max_batch_size_show, wq_max_batch_size_store);
1075
wq_ats_disable_show(struct device * dev,struct device_attribute * attr,char * buf)1076 static ssize_t wq_ats_disable_show(struct device *dev, struct device_attribute *attr, char *buf)
1077 {
1078 struct idxd_wq *wq = confdev_to_wq(dev);
1079
1080 return sysfs_emit(buf, "%u\n", test_bit(WQ_FLAG_ATS_DISABLE, &wq->flags));
1081 }
1082
wq_ats_disable_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1083 static ssize_t wq_ats_disable_store(struct device *dev, struct device_attribute *attr,
1084 const char *buf, size_t count)
1085 {
1086 struct idxd_wq *wq = confdev_to_wq(dev);
1087 struct idxd_device *idxd = wq->idxd;
1088 bool ats_dis;
1089 int rc;
1090
1091 if (wq->state != IDXD_WQ_DISABLED)
1092 return -EPERM;
1093
1094 if (!idxd->hw.wq_cap.wq_ats_support)
1095 return -EOPNOTSUPP;
1096
1097 rc = kstrtobool(buf, &ats_dis);
1098 if (rc < 0)
1099 return rc;
1100
1101 if (ats_dis)
1102 set_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
1103 else
1104 clear_bit(WQ_FLAG_ATS_DISABLE, &wq->flags);
1105
1106 return count;
1107 }
1108
1109 static struct device_attribute dev_attr_wq_ats_disable =
1110 __ATTR(ats_disable, 0644, wq_ats_disable_show, wq_ats_disable_store);
1111
wq_occupancy_show(struct device * dev,struct device_attribute * attr,char * buf)1112 static ssize_t wq_occupancy_show(struct device *dev, struct device_attribute *attr, char *buf)
1113 {
1114 struct idxd_wq *wq = confdev_to_wq(dev);
1115 struct idxd_device *idxd = wq->idxd;
1116 u32 occup, offset;
1117
1118 if (!idxd->hw.wq_cap.occupancy)
1119 return -EOPNOTSUPP;
1120
1121 offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_OCCUP_IDX);
1122 occup = ioread32(idxd->reg_base + offset) & WQCFG_OCCUP_MASK;
1123
1124 return sysfs_emit(buf, "%u\n", occup);
1125 }
1126
1127 static struct device_attribute dev_attr_wq_occupancy =
1128 __ATTR(occupancy, 0444, wq_occupancy_show, NULL);
1129
wq_enqcmds_retries_show(struct device * dev,struct device_attribute * attr,char * buf)1130 static ssize_t wq_enqcmds_retries_show(struct device *dev,
1131 struct device_attribute *attr, char *buf)
1132 {
1133 struct idxd_wq *wq = confdev_to_wq(dev);
1134
1135 if (wq_dedicated(wq))
1136 return -EOPNOTSUPP;
1137
1138 return sysfs_emit(buf, "%u\n", wq->enqcmds_retries);
1139 }
1140
wq_enqcmds_retries_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1141 static ssize_t wq_enqcmds_retries_store(struct device *dev, struct device_attribute *attr,
1142 const char *buf, size_t count)
1143 {
1144 struct idxd_wq *wq = confdev_to_wq(dev);
1145 int rc;
1146 unsigned int retries;
1147
1148 if (wq_dedicated(wq))
1149 return -EOPNOTSUPP;
1150
1151 rc = kstrtouint(buf, 10, &retries);
1152 if (rc < 0)
1153 return rc;
1154
1155 if (retries > IDXD_ENQCMDS_MAX_RETRIES)
1156 retries = IDXD_ENQCMDS_MAX_RETRIES;
1157
1158 wq->enqcmds_retries = retries;
1159 return count;
1160 }
1161
1162 static struct device_attribute dev_attr_wq_enqcmds_retries =
1163 __ATTR(enqcmds_retries, 0644, wq_enqcmds_retries_show, wq_enqcmds_retries_store);
1164
wq_op_config_show(struct device * dev,struct device_attribute * attr,char * buf)1165 static ssize_t wq_op_config_show(struct device *dev,
1166 struct device_attribute *attr, char *buf)
1167 {
1168 struct idxd_wq *wq = confdev_to_wq(dev);
1169
1170 return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, wq->opcap_bmap);
1171 }
1172
idxd_verify_supported_opcap(struct idxd_device * idxd,unsigned long * opmask)1173 static int idxd_verify_supported_opcap(struct idxd_device *idxd, unsigned long *opmask)
1174 {
1175 int bit;
1176
1177 /*
1178 * The OPCAP is defined as 256 bits that represents each operation the device
1179 * supports per bit. Iterate through all the bits and check if the input mask
1180 * is set for bits that are not set in the OPCAP for the device. If no OPCAP
1181 * bit is set and input mask has the bit set, then return error.
1182 */
1183 for_each_set_bit(bit, opmask, IDXD_MAX_OPCAP_BITS) {
1184 if (!test_bit(bit, idxd->opcap_bmap))
1185 return -EINVAL;
1186 }
1187
1188 return 0;
1189 }
1190
wq_op_config_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1191 static ssize_t wq_op_config_store(struct device *dev, struct device_attribute *attr,
1192 const char *buf, size_t count)
1193 {
1194 struct idxd_wq *wq = confdev_to_wq(dev);
1195 struct idxd_device *idxd = wq->idxd;
1196 unsigned long *opmask;
1197 int rc;
1198
1199 if (wq->state != IDXD_WQ_DISABLED)
1200 return -EPERM;
1201
1202 opmask = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL);
1203 if (!opmask)
1204 return -ENOMEM;
1205
1206 rc = bitmap_parse(buf, count, opmask, IDXD_MAX_OPCAP_BITS);
1207 if (rc < 0)
1208 goto err;
1209
1210 rc = idxd_verify_supported_opcap(idxd, opmask);
1211 if (rc < 0)
1212 goto err;
1213
1214 bitmap_copy(wq->opcap_bmap, opmask, IDXD_MAX_OPCAP_BITS);
1215
1216 bitmap_free(opmask);
1217 return count;
1218
1219 err:
1220 bitmap_free(opmask);
1221 return rc;
1222 }
1223
1224 static struct device_attribute dev_attr_wq_op_config =
1225 __ATTR(op_config, 0644, wq_op_config_show, wq_op_config_store);
1226
1227 static struct attribute *idxd_wq_attributes[] = {
1228 &dev_attr_wq_clients.attr,
1229 &dev_attr_wq_state.attr,
1230 &dev_attr_wq_group_id.attr,
1231 &dev_attr_wq_mode.attr,
1232 &dev_attr_wq_size.attr,
1233 &dev_attr_wq_priority.attr,
1234 &dev_attr_wq_block_on_fault.attr,
1235 &dev_attr_wq_threshold.attr,
1236 &dev_attr_wq_type.attr,
1237 &dev_attr_wq_name.attr,
1238 &dev_attr_wq_cdev_minor.attr,
1239 &dev_attr_wq_max_transfer_size.attr,
1240 &dev_attr_wq_max_batch_size.attr,
1241 &dev_attr_wq_ats_disable.attr,
1242 &dev_attr_wq_occupancy.attr,
1243 &dev_attr_wq_enqcmds_retries.attr,
1244 &dev_attr_wq_op_config.attr,
1245 NULL,
1246 };
1247
idxd_wq_attr_op_config_invisible(struct attribute * attr,struct idxd_device * idxd)1248 static bool idxd_wq_attr_op_config_invisible(struct attribute *attr,
1249 struct idxd_device *idxd)
1250 {
1251 return attr == &dev_attr_wq_op_config.attr &&
1252 !idxd->hw.wq_cap.op_config;
1253 }
1254
idxd_wq_attr_max_batch_size_invisible(struct attribute * attr,struct idxd_device * idxd)1255 static bool idxd_wq_attr_max_batch_size_invisible(struct attribute *attr,
1256 struct idxd_device *idxd)
1257 {
1258 /* Intel IAA does not support batch processing, make it invisible */
1259 return attr == &dev_attr_wq_max_batch_size.attr &&
1260 idxd->data->type == IDXD_TYPE_IAX;
1261 }
1262
idxd_wq_attr_visible(struct kobject * kobj,struct attribute * attr,int n)1263 static umode_t idxd_wq_attr_visible(struct kobject *kobj,
1264 struct attribute *attr, int n)
1265 {
1266 struct device *dev = container_of(kobj, struct device, kobj);
1267 struct idxd_wq *wq = confdev_to_wq(dev);
1268 struct idxd_device *idxd = wq->idxd;
1269
1270 if (idxd_wq_attr_op_config_invisible(attr, idxd))
1271 return 0;
1272
1273 if (idxd_wq_attr_max_batch_size_invisible(attr, idxd))
1274 return 0;
1275
1276 return attr->mode;
1277 }
1278
1279 static const struct attribute_group idxd_wq_attribute_group = {
1280 .attrs = idxd_wq_attributes,
1281 .is_visible = idxd_wq_attr_visible,
1282 };
1283
1284 static const struct attribute_group *idxd_wq_attribute_groups[] = {
1285 &idxd_wq_attribute_group,
1286 NULL,
1287 };
1288
idxd_conf_wq_release(struct device * dev)1289 static void idxd_conf_wq_release(struct device *dev)
1290 {
1291 struct idxd_wq *wq = confdev_to_wq(dev);
1292
1293 bitmap_free(wq->opcap_bmap);
1294 kfree(wq->wqcfg);
1295 kfree(wq);
1296 }
1297
1298 struct device_type idxd_wq_device_type = {
1299 .name = "wq",
1300 .release = idxd_conf_wq_release,
1301 .groups = idxd_wq_attribute_groups,
1302 };
1303
1304 /* IDXD device attribs */
version_show(struct device * dev,struct device_attribute * attr,char * buf)1305 static ssize_t version_show(struct device *dev, struct device_attribute *attr,
1306 char *buf)
1307 {
1308 struct idxd_device *idxd = confdev_to_idxd(dev);
1309
1310 return sysfs_emit(buf, "%#x\n", idxd->hw.version);
1311 }
1312 static DEVICE_ATTR_RO(version);
1313
max_work_queues_size_show(struct device * dev,struct device_attribute * attr,char * buf)1314 static ssize_t max_work_queues_size_show(struct device *dev,
1315 struct device_attribute *attr,
1316 char *buf)
1317 {
1318 struct idxd_device *idxd = confdev_to_idxd(dev);
1319
1320 return sysfs_emit(buf, "%u\n", idxd->max_wq_size);
1321 }
1322 static DEVICE_ATTR_RO(max_work_queues_size);
1323
max_groups_show(struct device * dev,struct device_attribute * attr,char * buf)1324 static ssize_t max_groups_show(struct device *dev,
1325 struct device_attribute *attr, char *buf)
1326 {
1327 struct idxd_device *idxd = confdev_to_idxd(dev);
1328
1329 return sysfs_emit(buf, "%u\n", idxd->max_groups);
1330 }
1331 static DEVICE_ATTR_RO(max_groups);
1332
max_work_queues_show(struct device * dev,struct device_attribute * attr,char * buf)1333 static ssize_t max_work_queues_show(struct device *dev,
1334 struct device_attribute *attr, char *buf)
1335 {
1336 struct idxd_device *idxd = confdev_to_idxd(dev);
1337
1338 return sysfs_emit(buf, "%u\n", idxd->max_wqs);
1339 }
1340 static DEVICE_ATTR_RO(max_work_queues);
1341
max_engines_show(struct device * dev,struct device_attribute * attr,char * buf)1342 static ssize_t max_engines_show(struct device *dev,
1343 struct device_attribute *attr, char *buf)
1344 {
1345 struct idxd_device *idxd = confdev_to_idxd(dev);
1346
1347 return sysfs_emit(buf, "%u\n", idxd->max_engines);
1348 }
1349 static DEVICE_ATTR_RO(max_engines);
1350
numa_node_show(struct device * dev,struct device_attribute * attr,char * buf)1351 static ssize_t numa_node_show(struct device *dev,
1352 struct device_attribute *attr, char *buf)
1353 {
1354 struct idxd_device *idxd = confdev_to_idxd(dev);
1355
1356 return sysfs_emit(buf, "%d\n", dev_to_node(&idxd->pdev->dev));
1357 }
1358 static DEVICE_ATTR_RO(numa_node);
1359
max_batch_size_show(struct device * dev,struct device_attribute * attr,char * buf)1360 static ssize_t max_batch_size_show(struct device *dev,
1361 struct device_attribute *attr, char *buf)
1362 {
1363 struct idxd_device *idxd = confdev_to_idxd(dev);
1364
1365 return sysfs_emit(buf, "%u\n", idxd->max_batch_size);
1366 }
1367 static DEVICE_ATTR_RO(max_batch_size);
1368
max_transfer_size_show(struct device * dev,struct device_attribute * attr,char * buf)1369 static ssize_t max_transfer_size_show(struct device *dev,
1370 struct device_attribute *attr,
1371 char *buf)
1372 {
1373 struct idxd_device *idxd = confdev_to_idxd(dev);
1374
1375 return sysfs_emit(buf, "%llu\n", idxd->max_xfer_bytes);
1376 }
1377 static DEVICE_ATTR_RO(max_transfer_size);
1378
op_cap_show(struct device * dev,struct device_attribute * attr,char * buf)1379 static ssize_t op_cap_show(struct device *dev,
1380 struct device_attribute *attr, char *buf)
1381 {
1382 struct idxd_device *idxd = confdev_to_idxd(dev);
1383
1384 return sysfs_emit(buf, "%*pb\n", IDXD_MAX_OPCAP_BITS, idxd->opcap_bmap);
1385 }
1386 static DEVICE_ATTR_RO(op_cap);
1387
gen_cap_show(struct device * dev,struct device_attribute * attr,char * buf)1388 static ssize_t gen_cap_show(struct device *dev,
1389 struct device_attribute *attr, char *buf)
1390 {
1391 struct idxd_device *idxd = confdev_to_idxd(dev);
1392
1393 return sysfs_emit(buf, "%#llx\n", idxd->hw.gen_cap.bits);
1394 }
1395 static DEVICE_ATTR_RO(gen_cap);
1396
configurable_show(struct device * dev,struct device_attribute * attr,char * buf)1397 static ssize_t configurable_show(struct device *dev,
1398 struct device_attribute *attr, char *buf)
1399 {
1400 struct idxd_device *idxd = confdev_to_idxd(dev);
1401
1402 return sysfs_emit(buf, "%u\n", test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags));
1403 }
1404 static DEVICE_ATTR_RO(configurable);
1405
clients_show(struct device * dev,struct device_attribute * attr,char * buf)1406 static ssize_t clients_show(struct device *dev,
1407 struct device_attribute *attr, char *buf)
1408 {
1409 struct idxd_device *idxd = confdev_to_idxd(dev);
1410 int count = 0, i;
1411
1412 spin_lock(&idxd->dev_lock);
1413 for (i = 0; i < idxd->max_wqs; i++) {
1414 struct idxd_wq *wq = idxd->wqs[i];
1415
1416 count += wq->client_count;
1417 }
1418 spin_unlock(&idxd->dev_lock);
1419
1420 return sysfs_emit(buf, "%d\n", count);
1421 }
1422 static DEVICE_ATTR_RO(clients);
1423
pasid_enabled_show(struct device * dev,struct device_attribute * attr,char * buf)1424 static ssize_t pasid_enabled_show(struct device *dev,
1425 struct device_attribute *attr, char *buf)
1426 {
1427 struct idxd_device *idxd = confdev_to_idxd(dev);
1428
1429 return sysfs_emit(buf, "%u\n", device_pasid_enabled(idxd));
1430 }
1431 static DEVICE_ATTR_RO(pasid_enabled);
1432
state_show(struct device * dev,struct device_attribute * attr,char * buf)1433 static ssize_t state_show(struct device *dev,
1434 struct device_attribute *attr, char *buf)
1435 {
1436 struct idxd_device *idxd = confdev_to_idxd(dev);
1437
1438 switch (idxd->state) {
1439 case IDXD_DEV_DISABLED:
1440 return sysfs_emit(buf, "disabled\n");
1441 case IDXD_DEV_ENABLED:
1442 return sysfs_emit(buf, "enabled\n");
1443 case IDXD_DEV_HALTED:
1444 return sysfs_emit(buf, "halted\n");
1445 }
1446
1447 return sysfs_emit(buf, "unknown\n");
1448 }
1449 static DEVICE_ATTR_RO(state);
1450
errors_show(struct device * dev,struct device_attribute * attr,char * buf)1451 static ssize_t errors_show(struct device *dev,
1452 struct device_attribute *attr, char *buf)
1453 {
1454 struct idxd_device *idxd = confdev_to_idxd(dev);
1455 int i, out = 0;
1456
1457 spin_lock(&idxd->dev_lock);
1458 for (i = 0; i < 4; i++)
1459 out += sysfs_emit_at(buf, out, "%#018llx ", idxd->sw_err.bits[i]);
1460 spin_unlock(&idxd->dev_lock);
1461 out--;
1462 out += sysfs_emit_at(buf, out, "\n");
1463 return out;
1464 }
1465 static DEVICE_ATTR_RO(errors);
1466
max_read_buffers_show(struct device * dev,struct device_attribute * attr,char * buf)1467 static ssize_t max_read_buffers_show(struct device *dev,
1468 struct device_attribute *attr, char *buf)
1469 {
1470 struct idxd_device *idxd = confdev_to_idxd(dev);
1471
1472 return sysfs_emit(buf, "%u\n", idxd->max_rdbufs);
1473 }
1474
max_tokens_show(struct device * dev,struct device_attribute * attr,char * buf)1475 static ssize_t max_tokens_show(struct device *dev,
1476 struct device_attribute *attr, char *buf)
1477 {
1478 dev_warn_once(dev, "attribute deprecated, see max_read_buffers.\n");
1479 return max_read_buffers_show(dev, attr, buf);
1480 }
1481
1482 static DEVICE_ATTR_RO(max_tokens); /* deprecated */
1483 static DEVICE_ATTR_RO(max_read_buffers);
1484
read_buffer_limit_show(struct device * dev,struct device_attribute * attr,char * buf)1485 static ssize_t read_buffer_limit_show(struct device *dev,
1486 struct device_attribute *attr, char *buf)
1487 {
1488 struct idxd_device *idxd = confdev_to_idxd(dev);
1489
1490 return sysfs_emit(buf, "%u\n", idxd->rdbuf_limit);
1491 }
1492
token_limit_show(struct device * dev,struct device_attribute * attr,char * buf)1493 static ssize_t token_limit_show(struct device *dev,
1494 struct device_attribute *attr, char *buf)
1495 {
1496 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit.\n");
1497 return read_buffer_limit_show(dev, attr, buf);
1498 }
1499
read_buffer_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1500 static ssize_t read_buffer_limit_store(struct device *dev,
1501 struct device_attribute *attr,
1502 const char *buf, size_t count)
1503 {
1504 struct idxd_device *idxd = confdev_to_idxd(dev);
1505 unsigned long val;
1506 int rc;
1507
1508 rc = kstrtoul(buf, 10, &val);
1509 if (rc < 0)
1510 return -EINVAL;
1511
1512 if (idxd->state == IDXD_DEV_ENABLED)
1513 return -EPERM;
1514
1515 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
1516 return -EPERM;
1517
1518 if (!idxd->hw.group_cap.rdbuf_limit)
1519 return -EPERM;
1520
1521 if (val > idxd->hw.group_cap.total_rdbufs)
1522 return -EINVAL;
1523
1524 idxd->rdbuf_limit = val;
1525 return count;
1526 }
1527
token_limit_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1528 static ssize_t token_limit_store(struct device *dev,
1529 struct device_attribute *attr,
1530 const char *buf, size_t count)
1531 {
1532 dev_warn_once(dev, "attribute deprecated, see read_buffer_limit\n");
1533 return read_buffer_limit_store(dev, attr, buf, count);
1534 }
1535
1536 static DEVICE_ATTR_RW(token_limit); /* deprecated */
1537 static DEVICE_ATTR_RW(read_buffer_limit);
1538
cdev_major_show(struct device * dev,struct device_attribute * attr,char * buf)1539 static ssize_t cdev_major_show(struct device *dev,
1540 struct device_attribute *attr, char *buf)
1541 {
1542 struct idxd_device *idxd = confdev_to_idxd(dev);
1543
1544 return sysfs_emit(buf, "%u\n", idxd->major);
1545 }
1546 static DEVICE_ATTR_RO(cdev_major);
1547
cmd_status_show(struct device * dev,struct device_attribute * attr,char * buf)1548 static ssize_t cmd_status_show(struct device *dev,
1549 struct device_attribute *attr, char *buf)
1550 {
1551 struct idxd_device *idxd = confdev_to_idxd(dev);
1552
1553 return sysfs_emit(buf, "%#x\n", idxd->cmd_status);
1554 }
1555
cmd_status_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t count)1556 static ssize_t cmd_status_store(struct device *dev, struct device_attribute *attr,
1557 const char *buf, size_t count)
1558 {
1559 struct idxd_device *idxd = confdev_to_idxd(dev);
1560
1561 idxd->cmd_status = 0;
1562 return count;
1563 }
1564 static DEVICE_ATTR_RW(cmd_status);
1565
idxd_device_attr_max_batch_size_invisible(struct attribute * attr,struct idxd_device * idxd)1566 static bool idxd_device_attr_max_batch_size_invisible(struct attribute *attr,
1567 struct idxd_device *idxd)
1568 {
1569 /* Intel IAA does not support batch processing, make it invisible */
1570 return attr == &dev_attr_max_batch_size.attr &&
1571 idxd->data->type == IDXD_TYPE_IAX;
1572 }
1573
idxd_device_attr_read_buffers_invisible(struct attribute * attr,struct idxd_device * idxd)1574 static bool idxd_device_attr_read_buffers_invisible(struct attribute *attr,
1575 struct idxd_device *idxd)
1576 {
1577 /*
1578 * Intel IAA does not support Read Buffer allocation control,
1579 * make these attributes invisible.
1580 */
1581 return (attr == &dev_attr_max_tokens.attr ||
1582 attr == &dev_attr_max_read_buffers.attr ||
1583 attr == &dev_attr_token_limit.attr ||
1584 attr == &dev_attr_read_buffer_limit.attr) &&
1585 idxd->data->type == IDXD_TYPE_IAX;
1586 }
1587
idxd_device_attr_visible(struct kobject * kobj,struct attribute * attr,int n)1588 static umode_t idxd_device_attr_visible(struct kobject *kobj,
1589 struct attribute *attr, int n)
1590 {
1591 struct device *dev = container_of(kobj, struct device, kobj);
1592 struct idxd_device *idxd = confdev_to_idxd(dev);
1593
1594 if (idxd_device_attr_max_batch_size_invisible(attr, idxd))
1595 return 0;
1596
1597 if (idxd_device_attr_read_buffers_invisible(attr, idxd))
1598 return 0;
1599
1600 return attr->mode;
1601 }
1602
1603 static struct attribute *idxd_device_attributes[] = {
1604 &dev_attr_version.attr,
1605 &dev_attr_max_groups.attr,
1606 &dev_attr_max_work_queues.attr,
1607 &dev_attr_max_work_queues_size.attr,
1608 &dev_attr_max_engines.attr,
1609 &dev_attr_numa_node.attr,
1610 &dev_attr_max_batch_size.attr,
1611 &dev_attr_max_transfer_size.attr,
1612 &dev_attr_op_cap.attr,
1613 &dev_attr_gen_cap.attr,
1614 &dev_attr_configurable.attr,
1615 &dev_attr_clients.attr,
1616 &dev_attr_pasid_enabled.attr,
1617 &dev_attr_state.attr,
1618 &dev_attr_errors.attr,
1619 &dev_attr_max_tokens.attr,
1620 &dev_attr_max_read_buffers.attr,
1621 &dev_attr_token_limit.attr,
1622 &dev_attr_read_buffer_limit.attr,
1623 &dev_attr_cdev_major.attr,
1624 &dev_attr_cmd_status.attr,
1625 NULL,
1626 };
1627
1628 static const struct attribute_group idxd_device_attribute_group = {
1629 .attrs = idxd_device_attributes,
1630 .is_visible = idxd_device_attr_visible,
1631 };
1632
1633 static const struct attribute_group *idxd_attribute_groups[] = {
1634 &idxd_device_attribute_group,
1635 NULL,
1636 };
1637
idxd_conf_device_release(struct device * dev)1638 static void idxd_conf_device_release(struct device *dev)
1639 {
1640 struct idxd_device *idxd = confdev_to_idxd(dev);
1641
1642 kfree(idxd->groups);
1643 bitmap_free(idxd->wq_enable_map);
1644 kfree(idxd->wqs);
1645 kfree(idxd->engines);
1646 ida_free(&idxd_ida, idxd->id);
1647 bitmap_free(idxd->opcap_bmap);
1648 kfree(idxd);
1649 }
1650
1651 struct device_type dsa_device_type = {
1652 .name = "dsa",
1653 .release = idxd_conf_device_release,
1654 .groups = idxd_attribute_groups,
1655 };
1656
1657 struct device_type iax_device_type = {
1658 .name = "iax",
1659 .release = idxd_conf_device_release,
1660 .groups = idxd_attribute_groups,
1661 };
1662
idxd_register_engine_devices(struct idxd_device * idxd)1663 static int idxd_register_engine_devices(struct idxd_device *idxd)
1664 {
1665 struct idxd_engine *engine;
1666 int i, j, rc;
1667
1668 for (i = 0; i < idxd->max_engines; i++) {
1669 engine = idxd->engines[i];
1670 rc = device_add(engine_confdev(engine));
1671 if (rc < 0)
1672 goto cleanup;
1673 }
1674
1675 return 0;
1676
1677 cleanup:
1678 j = i - 1;
1679 for (; i < idxd->max_engines; i++) {
1680 engine = idxd->engines[i];
1681 put_device(engine_confdev(engine));
1682 }
1683
1684 while (j--) {
1685 engine = idxd->engines[j];
1686 device_unregister(engine_confdev(engine));
1687 }
1688 return rc;
1689 }
1690
idxd_register_group_devices(struct idxd_device * idxd)1691 static int idxd_register_group_devices(struct idxd_device *idxd)
1692 {
1693 struct idxd_group *group;
1694 int i, j, rc;
1695
1696 for (i = 0; i < idxd->max_groups; i++) {
1697 group = idxd->groups[i];
1698 rc = device_add(group_confdev(group));
1699 if (rc < 0)
1700 goto cleanup;
1701 }
1702
1703 return 0;
1704
1705 cleanup:
1706 j = i - 1;
1707 for (; i < idxd->max_groups; i++) {
1708 group = idxd->groups[i];
1709 put_device(group_confdev(group));
1710 }
1711
1712 while (j--) {
1713 group = idxd->groups[j];
1714 device_unregister(group_confdev(group));
1715 }
1716 return rc;
1717 }
1718
idxd_register_wq_devices(struct idxd_device * idxd)1719 static int idxd_register_wq_devices(struct idxd_device *idxd)
1720 {
1721 struct idxd_wq *wq;
1722 int i, rc, j;
1723
1724 for (i = 0; i < idxd->max_wqs; i++) {
1725 wq = idxd->wqs[i];
1726 rc = device_add(wq_confdev(wq));
1727 if (rc < 0)
1728 goto cleanup;
1729 }
1730
1731 return 0;
1732
1733 cleanup:
1734 j = i - 1;
1735 for (; i < idxd->max_wqs; i++) {
1736 wq = idxd->wqs[i];
1737 put_device(wq_confdev(wq));
1738 }
1739
1740 while (j--) {
1741 wq = idxd->wqs[j];
1742 device_unregister(wq_confdev(wq));
1743 }
1744 return rc;
1745 }
1746
idxd_register_devices(struct idxd_device * idxd)1747 int idxd_register_devices(struct idxd_device *idxd)
1748 {
1749 struct device *dev = &idxd->pdev->dev;
1750 int rc, i;
1751
1752 rc = device_add(idxd_confdev(idxd));
1753 if (rc < 0)
1754 return rc;
1755
1756 rc = idxd_register_wq_devices(idxd);
1757 if (rc < 0) {
1758 dev_dbg(dev, "WQ devices registering failed: %d\n", rc);
1759 goto err_wq;
1760 }
1761
1762 rc = idxd_register_engine_devices(idxd);
1763 if (rc < 0) {
1764 dev_dbg(dev, "Engine devices registering failed: %d\n", rc);
1765 goto err_engine;
1766 }
1767
1768 rc = idxd_register_group_devices(idxd);
1769 if (rc < 0) {
1770 dev_dbg(dev, "Group device registering failed: %d\n", rc);
1771 goto err_group;
1772 }
1773
1774 return 0;
1775
1776 err_group:
1777 for (i = 0; i < idxd->max_engines; i++)
1778 device_unregister(engine_confdev(idxd->engines[i]));
1779 err_engine:
1780 for (i = 0; i < idxd->max_wqs; i++)
1781 device_unregister(wq_confdev(idxd->wqs[i]));
1782 err_wq:
1783 device_del(idxd_confdev(idxd));
1784 return rc;
1785 }
1786
idxd_unregister_devices(struct idxd_device * idxd)1787 void idxd_unregister_devices(struct idxd_device *idxd)
1788 {
1789 int i;
1790
1791 for (i = 0; i < idxd->max_wqs; i++) {
1792 struct idxd_wq *wq = idxd->wqs[i];
1793
1794 device_unregister(wq_confdev(wq));
1795 }
1796
1797 for (i = 0; i < idxd->max_engines; i++) {
1798 struct idxd_engine *engine = idxd->engines[i];
1799
1800 device_unregister(engine_confdev(engine));
1801 }
1802
1803 for (i = 0; i < idxd->max_groups; i++) {
1804 struct idxd_group *group = idxd->groups[i];
1805
1806 device_unregister(group_confdev(group));
1807 }
1808 }
1809
idxd_register_bus_type(void)1810 int idxd_register_bus_type(void)
1811 {
1812 return bus_register(&dsa_bus_type);
1813 }
1814
idxd_unregister_bus_type(void)1815 void idxd_unregister_bus_type(void)
1816 {
1817 bus_unregister(&dsa_bus_type);
1818 }
1819