1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #ifndef _IDXD_H_
4 #define _IDXD_H_
5 
6 #include <linux/sbitmap.h>
7 #include <linux/dmaengine.h>
8 #include <linux/percpu-rwsem.h>
9 #include <linux/wait.h>
10 #include <linux/cdev.h>
11 #include <linux/idr.h>
12 #include <linux/pci.h>
13 #include <linux/ioasid.h>
14 #include <linux/bitmap.h>
15 #include <linux/perf_event.h>
16 #include <uapi/linux/idxd.h>
17 #include "registers.h"
18 
19 #define IDXD_DRIVER_VERSION	"1.00"
20 
21 extern struct kmem_cache *idxd_desc_pool;
22 extern bool tc_override;
23 
24 struct idxd_wq;
25 struct idxd_dev;
26 
27 enum idxd_dev_type {
28 	IDXD_DEV_NONE = -1,
29 	IDXD_DEV_DSA = 0,
30 	IDXD_DEV_IAX,
31 	IDXD_DEV_WQ,
32 	IDXD_DEV_GROUP,
33 	IDXD_DEV_ENGINE,
34 	IDXD_DEV_CDEV,
35 	IDXD_DEV_MAX_TYPE,
36 };
37 
38 struct idxd_dev {
39 	struct device conf_dev;
40 	enum idxd_dev_type type;
41 };
42 
43 #define IDXD_REG_TIMEOUT	50
44 #define IDXD_DRAIN_TIMEOUT	5000
45 
46 enum idxd_type {
47 	IDXD_TYPE_UNKNOWN = -1,
48 	IDXD_TYPE_DSA = 0,
49 	IDXD_TYPE_IAX,
50 	IDXD_TYPE_MAX,
51 };
52 
53 #define IDXD_NAME_SIZE		128
54 #define IDXD_PMU_EVENT_MAX	64
55 
56 #define IDXD_ENQCMDS_RETRIES		32
57 #define IDXD_ENQCMDS_MAX_RETRIES	64
58 
59 struct idxd_device_driver {
60 	const char *name;
61 	enum idxd_dev_type *type;
62 	int (*probe)(struct idxd_dev *idxd_dev);
63 	void (*remove)(struct idxd_dev *idxd_dev);
64 	struct device_driver drv;
65 };
66 
67 extern struct idxd_device_driver dsa_drv;
68 extern struct idxd_device_driver idxd_drv;
69 extern struct idxd_device_driver idxd_dmaengine_drv;
70 extern struct idxd_device_driver idxd_user_drv;
71 
72 #define INVALID_INT_HANDLE	-1
73 struct idxd_irq_entry {
74 	int id;
75 	int vector;
76 	struct llist_head pending_llist;
77 	struct list_head work_list;
78 	/*
79 	 * Lock to protect access between irq thread process descriptor
80 	 * and irq thread processing error descriptor.
81 	 */
82 	spinlock_t list_lock;
83 	int int_handle;
84 	ioasid_t pasid;
85 };
86 
87 struct idxd_group {
88 	struct idxd_dev idxd_dev;
89 	struct idxd_device *idxd;
90 	struct grpcfg grpcfg;
91 	int id;
92 	int num_engines;
93 	int num_wqs;
94 	bool use_rdbuf_limit;
95 	u8 rdbufs_allowed;
96 	u8 rdbufs_reserved;
97 	int tc_a;
98 	int tc_b;
99 	int desc_progress_limit;
100 	int batch_progress_limit;
101 };
102 
103 struct idxd_pmu {
104 	struct idxd_device *idxd;
105 
106 	struct perf_event *event_list[IDXD_PMU_EVENT_MAX];
107 	int n_events;
108 
109 	DECLARE_BITMAP(used_mask, IDXD_PMU_EVENT_MAX);
110 
111 	struct pmu pmu;
112 	char name[IDXD_NAME_SIZE];
113 	int cpu;
114 
115 	int n_counters;
116 	int counter_width;
117 	int n_event_categories;
118 
119 	bool per_counter_caps_supported;
120 	unsigned long supported_event_categories;
121 
122 	unsigned long supported_filters;
123 	int n_filters;
124 
125 	struct hlist_node cpuhp_node;
126 };
127 
128 #define IDXD_MAX_PRIORITY	0xf
129 
130 enum idxd_wq_state {
131 	IDXD_WQ_DISABLED = 0,
132 	IDXD_WQ_ENABLED,
133 };
134 
135 enum idxd_wq_flag {
136 	WQ_FLAG_DEDICATED = 0,
137 	WQ_FLAG_BLOCK_ON_FAULT,
138 	WQ_FLAG_ATS_DISABLE,
139 };
140 
141 enum idxd_wq_type {
142 	IDXD_WQT_NONE = 0,
143 	IDXD_WQT_KERNEL,
144 	IDXD_WQT_USER,
145 };
146 
147 struct idxd_cdev {
148 	struct idxd_wq *wq;
149 	struct cdev cdev;
150 	struct idxd_dev idxd_dev;
151 	int minor;
152 };
153 
154 #define IDXD_ALLOCATED_BATCH_SIZE	128U
155 #define WQ_NAME_SIZE   1024
156 #define WQ_TYPE_SIZE   10
157 
158 #define WQ_DEFAULT_QUEUE_DEPTH		16
159 #define WQ_DEFAULT_MAX_XFER		SZ_2M
160 #define WQ_DEFAULT_MAX_BATCH		32
161 
162 enum idxd_op_type {
163 	IDXD_OP_BLOCK = 0,
164 	IDXD_OP_NONBLOCK = 1,
165 };
166 
167 enum idxd_complete_type {
168 	IDXD_COMPLETE_NORMAL = 0,
169 	IDXD_COMPLETE_ABORT,
170 	IDXD_COMPLETE_DEV_FAIL,
171 };
172 
173 struct idxd_dma_chan {
174 	struct dma_chan chan;
175 	struct idxd_wq *wq;
176 };
177 
178 struct idxd_wq {
179 	void __iomem *portal;
180 	u32 portal_offset;
181 	unsigned int enqcmds_retries;
182 	struct percpu_ref wq_active;
183 	struct completion wq_dead;
184 	struct completion wq_resurrect;
185 	struct idxd_dev idxd_dev;
186 	struct idxd_cdev *idxd_cdev;
187 	struct wait_queue_head err_queue;
188 	struct idxd_device *idxd;
189 	int id;
190 	struct idxd_irq_entry ie;
191 	enum idxd_wq_type type;
192 	struct idxd_group *group;
193 	int client_count;
194 	struct mutex wq_lock;	/* mutex for workqueue */
195 	u32 size;
196 	u32 threshold;
197 	u32 priority;
198 	enum idxd_wq_state state;
199 	unsigned long flags;
200 	union wqcfg *wqcfg;
201 	unsigned long *opcap_bmap;
202 
203 	struct dsa_hw_desc **hw_descs;
204 	int num_descs;
205 	union {
206 		struct dsa_completion_record *compls;
207 		struct iax_completion_record *iax_compls;
208 	};
209 	dma_addr_t compls_addr;
210 	int compls_size;
211 	struct idxd_desc **descs;
212 	struct sbitmap_queue sbq;
213 	struct idxd_dma_chan *idxd_chan;
214 	char name[WQ_NAME_SIZE + 1];
215 	u64 max_xfer_bytes;
216 	u32 max_batch_size;
217 };
218 
219 struct idxd_engine {
220 	struct idxd_dev idxd_dev;
221 	int id;
222 	struct idxd_group *group;
223 	struct idxd_device *idxd;
224 };
225 
226 /* shadow registers */
227 struct idxd_hw {
228 	u32 version;
229 	union gen_cap_reg gen_cap;
230 	union wq_cap_reg wq_cap;
231 	union group_cap_reg group_cap;
232 	union engine_cap_reg engine_cap;
233 	struct opcap opcap;
234 	u32 cmd_cap;
235 };
236 
237 enum idxd_device_state {
238 	IDXD_DEV_HALTED = -1,
239 	IDXD_DEV_DISABLED = 0,
240 	IDXD_DEV_ENABLED,
241 };
242 
243 enum idxd_device_flag {
244 	IDXD_FLAG_CONFIGURABLE = 0,
245 	IDXD_FLAG_CMD_RUNNING,
246 	IDXD_FLAG_PASID_ENABLED,
247 	IDXD_FLAG_USER_PASID_ENABLED,
248 };
249 
250 struct idxd_dma_dev {
251 	struct idxd_device *idxd;
252 	struct dma_device dma;
253 };
254 
255 struct idxd_driver_data {
256 	const char *name_prefix;
257 	enum idxd_type type;
258 	struct device_type *dev_type;
259 	int compl_size;
260 	int align;
261 };
262 
263 struct idxd_device {
264 	struct idxd_dev idxd_dev;
265 	struct idxd_driver_data *data;
266 	struct list_head list;
267 	struct idxd_hw hw;
268 	enum idxd_device_state state;
269 	unsigned long flags;
270 	int id;
271 	int major;
272 	u32 cmd_status;
273 	struct idxd_irq_entry ie;	/* misc irq, msix 0 */
274 
275 	struct pci_dev *pdev;
276 	void __iomem *reg_base;
277 
278 	spinlock_t dev_lock;	/* spinlock for device */
279 	spinlock_t cmd_lock;	/* spinlock for device commands */
280 	struct completion *cmd_done;
281 	struct idxd_group **groups;
282 	struct idxd_wq **wqs;
283 	struct idxd_engine **engines;
284 
285 	struct iommu_sva *sva;
286 	unsigned int pasid;
287 
288 	int num_groups;
289 	int irq_cnt;
290 	bool request_int_handles;
291 
292 	u32 msix_perm_offset;
293 	u32 wqcfg_offset;
294 	u32 grpcfg_offset;
295 	u32 perfmon_offset;
296 
297 	u64 max_xfer_bytes;
298 	u32 max_batch_size;
299 	int max_groups;
300 	int max_engines;
301 	int max_rdbufs;
302 	int max_wqs;
303 	int max_wq_size;
304 	int rdbuf_limit;
305 	int nr_rdbufs;		/* non-reserved read buffers */
306 	unsigned int wqcfg_size;
307 	unsigned long *wq_enable_map;
308 
309 	union sw_err_reg sw_err;
310 	wait_queue_head_t cmd_waitq;
311 
312 	struct idxd_dma_dev *idxd_dma;
313 	struct workqueue_struct *wq;
314 	struct work_struct work;
315 
316 	struct idxd_pmu *idxd_pmu;
317 
318 	unsigned long *opcap_bmap;
319 };
320 
321 /* IDXD software descriptor */
322 struct idxd_desc {
323 	union {
324 		struct dsa_hw_desc *hw;
325 		struct iax_hw_desc *iax_hw;
326 	};
327 	dma_addr_t desc_dma;
328 	union {
329 		struct dsa_completion_record *completion;
330 		struct iax_completion_record *iax_completion;
331 	};
332 	dma_addr_t compl_dma;
333 	struct dma_async_tx_descriptor txd;
334 	struct llist_node llnode;
335 	struct list_head list;
336 	int id;
337 	int cpu;
338 	struct idxd_wq *wq;
339 };
340 
341 /*
342  * This is software defined error for the completion status. We overload the error code
343  * that will never appear in completion status and only SWERR register.
344  */
345 enum idxd_completion_status {
346 	IDXD_COMP_DESC_ABORT = 0xff,
347 };
348 
349 #define idxd_confdev(idxd) &idxd->idxd_dev.conf_dev
350 #define wq_confdev(wq) &wq->idxd_dev.conf_dev
351 #define engine_confdev(engine) &engine->idxd_dev.conf_dev
352 #define group_confdev(group) &group->idxd_dev.conf_dev
353 #define cdev_dev(cdev) &cdev->idxd_dev.conf_dev
354 
355 #define confdev_to_idxd_dev(dev) container_of(dev, struct idxd_dev, conf_dev)
356 #define idxd_dev_to_idxd(idxd_dev) container_of(idxd_dev, struct idxd_device, idxd_dev)
357 #define idxd_dev_to_wq(idxd_dev) container_of(idxd_dev, struct idxd_wq, idxd_dev)
358 
confdev_to_idxd(struct device * dev)359 static inline struct idxd_device *confdev_to_idxd(struct device *dev)
360 {
361 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
362 
363 	return idxd_dev_to_idxd(idxd_dev);
364 }
365 
confdev_to_wq(struct device * dev)366 static inline struct idxd_wq *confdev_to_wq(struct device *dev)
367 {
368 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
369 
370 	return idxd_dev_to_wq(idxd_dev);
371 }
372 
confdev_to_engine(struct device * dev)373 static inline struct idxd_engine *confdev_to_engine(struct device *dev)
374 {
375 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
376 
377 	return container_of(idxd_dev, struct idxd_engine, idxd_dev);
378 }
379 
confdev_to_group(struct device * dev)380 static inline struct idxd_group *confdev_to_group(struct device *dev)
381 {
382 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
383 
384 	return container_of(idxd_dev, struct idxd_group, idxd_dev);
385 }
386 
dev_to_cdev(struct device * dev)387 static inline struct idxd_cdev *dev_to_cdev(struct device *dev)
388 {
389 	struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
390 
391 	return container_of(idxd_dev, struct idxd_cdev, idxd_dev);
392 }
393 
idxd_dev_set_type(struct idxd_dev * idev,int type)394 static inline void idxd_dev_set_type(struct idxd_dev *idev, int type)
395 {
396 	if (type >= IDXD_DEV_MAX_TYPE) {
397 		idev->type = IDXD_DEV_NONE;
398 		return;
399 	}
400 
401 	idev->type = type;
402 }
403 
idxd_get_ie(struct idxd_device * idxd,int idx)404 static inline struct idxd_irq_entry *idxd_get_ie(struct idxd_device *idxd, int idx)
405 {
406 	return (idx == 0) ? &idxd->ie : &idxd->wqs[idx - 1]->ie;
407 }
408 
ie_to_wq(struct idxd_irq_entry * ie)409 static inline struct idxd_wq *ie_to_wq(struct idxd_irq_entry *ie)
410 {
411 	return container_of(ie, struct idxd_wq, ie);
412 }
413 
ie_to_idxd(struct idxd_irq_entry * ie)414 static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
415 {
416 	return container_of(ie, struct idxd_device, ie);
417 }
418 
419 extern struct bus_type dsa_bus_type;
420 
421 extern bool support_enqcmd;
422 extern struct ida idxd_ida;
423 extern struct device_type dsa_device_type;
424 extern struct device_type iax_device_type;
425 extern struct device_type idxd_wq_device_type;
426 extern struct device_type idxd_engine_device_type;
427 extern struct device_type idxd_group_device_type;
428 
is_dsa_dev(struct idxd_dev * idxd_dev)429 static inline bool is_dsa_dev(struct idxd_dev *idxd_dev)
430 {
431 	return idxd_dev->type == IDXD_DEV_DSA;
432 }
433 
is_iax_dev(struct idxd_dev * idxd_dev)434 static inline bool is_iax_dev(struct idxd_dev *idxd_dev)
435 {
436 	return idxd_dev->type == IDXD_DEV_IAX;
437 }
438 
is_idxd_dev(struct idxd_dev * idxd_dev)439 static inline bool is_idxd_dev(struct idxd_dev *idxd_dev)
440 {
441 	return is_dsa_dev(idxd_dev) || is_iax_dev(idxd_dev);
442 }
443 
is_idxd_wq_dev(struct idxd_dev * idxd_dev)444 static inline bool is_idxd_wq_dev(struct idxd_dev *idxd_dev)
445 {
446 	return idxd_dev->type == IDXD_DEV_WQ;
447 }
448 
is_idxd_wq_dmaengine(struct idxd_wq * wq)449 static inline bool is_idxd_wq_dmaengine(struct idxd_wq *wq)
450 {
451 	if (wq->type == IDXD_WQT_KERNEL && strcmp(wq->name, "dmaengine") == 0)
452 		return true;
453 	return false;
454 }
455 
is_idxd_wq_user(struct idxd_wq * wq)456 static inline bool is_idxd_wq_user(struct idxd_wq *wq)
457 {
458 	return wq->type == IDXD_WQT_USER;
459 }
460 
is_idxd_wq_kernel(struct idxd_wq * wq)461 static inline bool is_idxd_wq_kernel(struct idxd_wq *wq)
462 {
463 	return wq->type == IDXD_WQT_KERNEL;
464 }
465 
wq_dedicated(struct idxd_wq * wq)466 static inline bool wq_dedicated(struct idxd_wq *wq)
467 {
468 	return test_bit(WQ_FLAG_DEDICATED, &wq->flags);
469 }
470 
wq_shared(struct idxd_wq * wq)471 static inline bool wq_shared(struct idxd_wq *wq)
472 {
473 	return !test_bit(WQ_FLAG_DEDICATED, &wq->flags);
474 }
475 
device_pasid_enabled(struct idxd_device * idxd)476 static inline bool device_pasid_enabled(struct idxd_device *idxd)
477 {
478 	return test_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
479 }
480 
device_user_pasid_enabled(struct idxd_device * idxd)481 static inline bool device_user_pasid_enabled(struct idxd_device *idxd)
482 {
483 	return test_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
484 }
485 
wq_pasid_enabled(struct idxd_wq * wq)486 static inline bool wq_pasid_enabled(struct idxd_wq *wq)
487 {
488 	return (is_idxd_wq_kernel(wq) && device_pasid_enabled(wq->idxd)) ||
489 	       (is_idxd_wq_user(wq) && device_user_pasid_enabled(wq->idxd));
490 }
491 
wq_shared_supported(struct idxd_wq * wq)492 static inline bool wq_shared_supported(struct idxd_wq *wq)
493 {
494 	return (support_enqcmd && wq_pasid_enabled(wq));
495 }
496 
497 enum idxd_portal_prot {
498 	IDXD_PORTAL_UNLIMITED = 0,
499 	IDXD_PORTAL_LIMITED,
500 };
501 
502 enum idxd_interrupt_type {
503 	IDXD_IRQ_MSIX = 0,
504 	IDXD_IRQ_IMS,
505 };
506 
idxd_get_wq_portal_offset(enum idxd_portal_prot prot)507 static inline int idxd_get_wq_portal_offset(enum idxd_portal_prot prot)
508 {
509 	return prot * 0x1000;
510 }
511 
idxd_get_wq_portal_full_offset(int wq_id,enum idxd_portal_prot prot)512 static inline int idxd_get_wq_portal_full_offset(int wq_id,
513 						 enum idxd_portal_prot prot)
514 {
515 	return ((wq_id * 4) << PAGE_SHIFT) + idxd_get_wq_portal_offset(prot);
516 }
517 
518 #define IDXD_PORTAL_MASK	(PAGE_SIZE - 1)
519 
520 /*
521  * Even though this function can be accessed by multiple threads, it is safe to use.
522  * At worst the address gets used more than once before it gets incremented. We don't
523  * hit a threshold until iops becomes many million times a second. So the occasional
524  * reuse of the same address is tolerable compare to using an atomic variable. This is
525  * safe on a system that has atomic load/store for 32bit integers. Given that this is an
526  * Intel iEP device, that should not be a problem.
527  */
idxd_wq_portal_addr(struct idxd_wq * wq)528 static inline void __iomem *idxd_wq_portal_addr(struct idxd_wq *wq)
529 {
530 	int ofs = wq->portal_offset;
531 
532 	wq->portal_offset = (ofs + sizeof(struct dsa_raw_desc)) & IDXD_PORTAL_MASK;
533 	return wq->portal + ofs;
534 }
535 
idxd_wq_get(struct idxd_wq * wq)536 static inline void idxd_wq_get(struct idxd_wq *wq)
537 {
538 	wq->client_count++;
539 }
540 
idxd_wq_put(struct idxd_wq * wq)541 static inline void idxd_wq_put(struct idxd_wq *wq)
542 {
543 	wq->client_count--;
544 }
545 
idxd_wq_refcount(struct idxd_wq * wq)546 static inline int idxd_wq_refcount(struct idxd_wq *wq)
547 {
548 	return wq->client_count;
549 };
550 
551 /*
552  * Intel IAA does not support batch processing.
553  * The max batch size of device, max batch size of wq and
554  * max batch shift of wqcfg should be always 0 on IAA.
555  */
idxd_set_max_batch_size(int idxd_type,struct idxd_device * idxd,u32 max_batch_size)556 static inline void idxd_set_max_batch_size(int idxd_type, struct idxd_device *idxd,
557 					   u32 max_batch_size)
558 {
559 	if (idxd_type == IDXD_TYPE_IAX)
560 		idxd->max_batch_size = 0;
561 	else
562 		idxd->max_batch_size = max_batch_size;
563 }
564 
idxd_wq_set_max_batch_size(int idxd_type,struct idxd_wq * wq,u32 max_batch_size)565 static inline void idxd_wq_set_max_batch_size(int idxd_type, struct idxd_wq *wq,
566 					      u32 max_batch_size)
567 {
568 	if (idxd_type == IDXD_TYPE_IAX)
569 		wq->max_batch_size = 0;
570 	else
571 		wq->max_batch_size = max_batch_size;
572 }
573 
idxd_wqcfg_set_max_batch_shift(int idxd_type,union wqcfg * wqcfg,u32 max_batch_shift)574 static inline void idxd_wqcfg_set_max_batch_shift(int idxd_type, union wqcfg *wqcfg,
575 						  u32 max_batch_shift)
576 {
577 	if (idxd_type == IDXD_TYPE_IAX)
578 		wqcfg->max_batch_shift = 0;
579 	else
580 		wqcfg->max_batch_shift = max_batch_shift;
581 }
582 
583 int __must_check __idxd_driver_register(struct idxd_device_driver *idxd_drv,
584 					struct module *module, const char *mod_name);
585 #define idxd_driver_register(driver) \
586 	__idxd_driver_register(driver, THIS_MODULE, KBUILD_MODNAME)
587 
588 void idxd_driver_unregister(struct idxd_device_driver *idxd_drv);
589 
590 #define module_idxd_driver(__idxd_driver) \
591 	module_driver(__idxd_driver, idxd_driver_register, idxd_driver_unregister)
592 
593 int idxd_register_bus_type(void);
594 void idxd_unregister_bus_type(void);
595 int idxd_register_devices(struct idxd_device *idxd);
596 void idxd_unregister_devices(struct idxd_device *idxd);
597 int idxd_register_driver(void);
598 void idxd_unregister_driver(void);
599 void idxd_wqs_quiesce(struct idxd_device *idxd);
600 bool idxd_queue_int_handle_resubmit(struct idxd_desc *desc);
601 
602 /* device interrupt control */
603 irqreturn_t idxd_misc_thread(int vec, void *data);
604 irqreturn_t idxd_wq_thread(int irq, void *data);
605 void idxd_mask_error_interrupts(struct idxd_device *idxd);
606 void idxd_unmask_error_interrupts(struct idxd_device *idxd);
607 
608 /* device control */
609 int idxd_register_idxd_drv(void);
610 void idxd_unregister_idxd_drv(void);
611 int idxd_device_drv_probe(struct idxd_dev *idxd_dev);
612 void idxd_device_drv_remove(struct idxd_dev *idxd_dev);
613 int drv_enable_wq(struct idxd_wq *wq);
614 void drv_disable_wq(struct idxd_wq *wq);
615 int idxd_device_init_reset(struct idxd_device *idxd);
616 int idxd_device_enable(struct idxd_device *idxd);
617 int idxd_device_disable(struct idxd_device *idxd);
618 void idxd_device_reset(struct idxd_device *idxd);
619 void idxd_device_clear_state(struct idxd_device *idxd);
620 int idxd_device_config(struct idxd_device *idxd);
621 void idxd_device_drain_pasid(struct idxd_device *idxd, int pasid);
622 int idxd_device_load_config(struct idxd_device *idxd);
623 int idxd_device_request_int_handle(struct idxd_device *idxd, int idx, int *handle,
624 				   enum idxd_interrupt_type irq_type);
625 int idxd_device_release_int_handle(struct idxd_device *idxd, int handle,
626 				   enum idxd_interrupt_type irq_type);
627 
628 /* work queue control */
629 void idxd_wqs_unmap_portal(struct idxd_device *idxd);
630 int idxd_wq_alloc_resources(struct idxd_wq *wq);
631 void idxd_wq_free_resources(struct idxd_wq *wq);
632 int idxd_wq_enable(struct idxd_wq *wq);
633 int idxd_wq_disable(struct idxd_wq *wq, bool reset_config);
634 void idxd_wq_drain(struct idxd_wq *wq);
635 void idxd_wq_reset(struct idxd_wq *wq);
636 int idxd_wq_map_portal(struct idxd_wq *wq);
637 void idxd_wq_unmap_portal(struct idxd_wq *wq);
638 int idxd_wq_set_pasid(struct idxd_wq *wq, int pasid);
639 int idxd_wq_disable_pasid(struct idxd_wq *wq);
640 void __idxd_wq_quiesce(struct idxd_wq *wq);
641 void idxd_wq_quiesce(struct idxd_wq *wq);
642 int idxd_wq_init_percpu_ref(struct idxd_wq *wq);
643 void idxd_wq_free_irq(struct idxd_wq *wq);
644 int idxd_wq_request_irq(struct idxd_wq *wq);
645 
646 /* submission */
647 int idxd_submit_desc(struct idxd_wq *wq, struct idxd_desc *desc);
648 struct idxd_desc *idxd_alloc_desc(struct idxd_wq *wq, enum idxd_op_type optype);
649 void idxd_free_desc(struct idxd_wq *wq, struct idxd_desc *desc);
650 int idxd_enqcmds(struct idxd_wq *wq, void __iomem *portal, const void *desc);
651 
652 /* dmaengine */
653 int idxd_register_dma_device(struct idxd_device *idxd);
654 void idxd_unregister_dma_device(struct idxd_device *idxd);
655 void idxd_parse_completion_status(u8 status, enum dmaengine_tx_result *res);
656 void idxd_dma_complete_txd(struct idxd_desc *desc,
657 			   enum idxd_complete_type comp_type, bool free_desc);
658 
659 /* cdev */
660 int idxd_cdev_register(void);
661 void idxd_cdev_remove(void);
662 int idxd_cdev_get_major(struct idxd_device *idxd);
663 int idxd_wq_add_cdev(struct idxd_wq *wq);
664 void idxd_wq_del_cdev(struct idxd_wq *wq);
665 
666 /* perfmon */
667 #if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
668 int perfmon_pmu_init(struct idxd_device *idxd);
669 void perfmon_pmu_remove(struct idxd_device *idxd);
670 void perfmon_counter_overflow(struct idxd_device *idxd);
671 void perfmon_init(void);
672 void perfmon_exit(void);
673 #else
perfmon_pmu_init(struct idxd_device * idxd)674 static inline int perfmon_pmu_init(struct idxd_device *idxd) { return 0; }
perfmon_pmu_remove(struct idxd_device * idxd)675 static inline void perfmon_pmu_remove(struct idxd_device *idxd) {}
perfmon_counter_overflow(struct idxd_device * idxd)676 static inline void perfmon_counter_overflow(struct idxd_device *idxd) {}
perfmon_init(void)677 static inline void perfmon_init(void) {}
perfmon_exit(void)678 static inline void perfmon_exit(void) {}
679 #endif
680 
681 #endif
682