1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/pci.h>
7 #include <linux/device.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/dmaengine.h>
10 #include <uapi/linux/idxd.h>
11 #include "../dmaengine.h"
12 #include "registers.h"
13 #include "idxd.h"
14 
to_idxd_wq(struct dma_chan * c)15 static inline struct idxd_wq *to_idxd_wq(struct dma_chan *c)
16 {
17 	struct idxd_dma_chan *idxd_chan;
18 
19 	idxd_chan = container_of(c, struct idxd_dma_chan, chan);
20 	return idxd_chan->wq;
21 }
22 
idxd_dma_complete_txd(struct idxd_desc * desc,enum idxd_complete_type comp_type,bool free_desc)23 void idxd_dma_complete_txd(struct idxd_desc *desc,
24 			   enum idxd_complete_type comp_type,
25 			   bool free_desc)
26 {
27 	struct idxd_device *idxd = desc->wq->idxd;
28 	struct dma_async_tx_descriptor *tx;
29 	struct dmaengine_result res;
30 	int complete = 1;
31 
32 	if (desc->completion->status == DSA_COMP_SUCCESS) {
33 		res.result = DMA_TRANS_NOERROR;
34 	} else if (desc->completion->status) {
35 		if (idxd->request_int_handles && comp_type != IDXD_COMPLETE_ABORT &&
36 		    desc->completion->status == DSA_COMP_INT_HANDLE_INVAL &&
37 		    idxd_queue_int_handle_resubmit(desc))
38 			return;
39 		res.result = DMA_TRANS_WRITE_FAILED;
40 	} else if (comp_type == IDXD_COMPLETE_ABORT) {
41 		res.result = DMA_TRANS_ABORTED;
42 	} else {
43 		complete = 0;
44 	}
45 
46 	tx = &desc->txd;
47 	if (complete && tx->cookie) {
48 		dma_cookie_complete(tx);
49 		dma_descriptor_unmap(tx);
50 		dmaengine_desc_get_callback_invoke(tx, &res);
51 		tx->callback = NULL;
52 		tx->callback_result = NULL;
53 	}
54 
55 	if (free_desc)
56 		idxd_free_desc(desc->wq, desc);
57 }
58 
op_flag_setup(unsigned long flags,u32 * desc_flags)59 static void op_flag_setup(unsigned long flags, u32 *desc_flags)
60 {
61 	*desc_flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
62 	if (flags & DMA_PREP_INTERRUPT)
63 		*desc_flags |= IDXD_OP_FLAG_RCI;
64 }
65 
set_completion_address(struct idxd_desc * desc,u64 * compl_addr)66 static inline void set_completion_address(struct idxd_desc *desc,
67 					  u64 *compl_addr)
68 {
69 		*compl_addr = desc->compl_dma;
70 }
71 
idxd_prep_desc_common(struct idxd_wq * wq,struct dsa_hw_desc * hw,char opcode,u64 addr_f1,u64 addr_f2,u64 len,u64 compl,u32 flags)72 static inline void idxd_prep_desc_common(struct idxd_wq *wq,
73 					 struct dsa_hw_desc *hw, char opcode,
74 					 u64 addr_f1, u64 addr_f2, u64 len,
75 					 u64 compl, u32 flags)
76 {
77 	hw->flags = flags;
78 	hw->opcode = opcode;
79 	hw->src_addr = addr_f1;
80 	hw->dst_addr = addr_f2;
81 	hw->xfer_size = len;
82 	/*
83 	 * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv
84 	 * field instead. This field should be set to 1 for kernel descriptors.
85 	 */
86 	hw->priv = 1;
87 	hw->completion_addr = compl;
88 }
89 
90 static struct dma_async_tx_descriptor *
idxd_dma_prep_interrupt(struct dma_chan * c,unsigned long flags)91 idxd_dma_prep_interrupt(struct dma_chan *c, unsigned long flags)
92 {
93 	struct idxd_wq *wq = to_idxd_wq(c);
94 	u32 desc_flags;
95 	struct idxd_desc *desc;
96 
97 	if (wq->state != IDXD_WQ_ENABLED)
98 		return NULL;
99 
100 	op_flag_setup(flags, &desc_flags);
101 	desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
102 	if (IS_ERR(desc))
103 		return NULL;
104 
105 	idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_NOOP,
106 			      0, 0, 0, desc->compl_dma, desc_flags);
107 	desc->txd.flags = flags;
108 	return &desc->txd;
109 }
110 
111 static struct dma_async_tx_descriptor *
idxd_dma_submit_memcpy(struct dma_chan * c,dma_addr_t dma_dest,dma_addr_t dma_src,size_t len,unsigned long flags)112 idxd_dma_submit_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
113 		       dma_addr_t dma_src, size_t len, unsigned long flags)
114 {
115 	struct idxd_wq *wq = to_idxd_wq(c);
116 	u32 desc_flags;
117 	struct idxd_device *idxd = wq->idxd;
118 	struct idxd_desc *desc;
119 
120 	if (wq->state != IDXD_WQ_ENABLED)
121 		return NULL;
122 
123 	if (len > idxd->max_xfer_bytes)
124 		return NULL;
125 
126 	op_flag_setup(flags, &desc_flags);
127 	desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
128 	if (IS_ERR(desc))
129 		return NULL;
130 
131 	idxd_prep_desc_common(wq, desc->hw, DSA_OPCODE_MEMMOVE,
132 			      dma_src, dma_dest, len, desc->compl_dma,
133 			      desc_flags);
134 
135 	desc->txd.flags = flags;
136 
137 	return &desc->txd;
138 }
139 
idxd_dma_alloc_chan_resources(struct dma_chan * chan)140 static int idxd_dma_alloc_chan_resources(struct dma_chan *chan)
141 {
142 	struct idxd_wq *wq = to_idxd_wq(chan);
143 	struct device *dev = &wq->idxd->pdev->dev;
144 
145 	idxd_wq_get(wq);
146 	dev_dbg(dev, "%s: client_count: %d\n", __func__,
147 		idxd_wq_refcount(wq));
148 	return 0;
149 }
150 
idxd_dma_free_chan_resources(struct dma_chan * chan)151 static void idxd_dma_free_chan_resources(struct dma_chan *chan)
152 {
153 	struct idxd_wq *wq = to_idxd_wq(chan);
154 	struct device *dev = &wq->idxd->pdev->dev;
155 
156 	idxd_wq_put(wq);
157 	dev_dbg(dev, "%s: client_count: %d\n", __func__,
158 		idxd_wq_refcount(wq));
159 }
160 
idxd_dma_tx_status(struct dma_chan * dma_chan,dma_cookie_t cookie,struct dma_tx_state * txstate)161 static enum dma_status idxd_dma_tx_status(struct dma_chan *dma_chan,
162 					  dma_cookie_t cookie,
163 					  struct dma_tx_state *txstate)
164 {
165 	return DMA_OUT_OF_ORDER;
166 }
167 
168 /*
169  * issue_pending() does not need to do anything since tx_submit() does the job
170  * already.
171  */
idxd_dma_issue_pending(struct dma_chan * dma_chan)172 static void idxd_dma_issue_pending(struct dma_chan *dma_chan)
173 {
174 }
175 
idxd_dma_tx_submit(struct dma_async_tx_descriptor * tx)176 static dma_cookie_t idxd_dma_tx_submit(struct dma_async_tx_descriptor *tx)
177 {
178 	struct dma_chan *c = tx->chan;
179 	struct idxd_wq *wq = to_idxd_wq(c);
180 	dma_cookie_t cookie;
181 	int rc;
182 	struct idxd_desc *desc = container_of(tx, struct idxd_desc, txd);
183 
184 	cookie = dma_cookie_assign(tx);
185 
186 	rc = idxd_submit_desc(wq, desc);
187 	if (rc < 0) {
188 		idxd_free_desc(wq, desc);
189 		return rc;
190 	}
191 
192 	return cookie;
193 }
194 
idxd_dma_release(struct dma_device * device)195 static void idxd_dma_release(struct dma_device *device)
196 {
197 	struct idxd_dma_dev *idxd_dma = container_of(device, struct idxd_dma_dev, dma);
198 
199 	kfree(idxd_dma);
200 }
201 
idxd_register_dma_device(struct idxd_device * idxd)202 int idxd_register_dma_device(struct idxd_device *idxd)
203 {
204 	struct idxd_dma_dev *idxd_dma;
205 	struct dma_device *dma;
206 	struct device *dev = &idxd->pdev->dev;
207 	int rc;
208 
209 	idxd_dma = kzalloc_node(sizeof(*idxd_dma), GFP_KERNEL, dev_to_node(dev));
210 	if (!idxd_dma)
211 		return -ENOMEM;
212 
213 	dma = &idxd_dma->dma;
214 	INIT_LIST_HEAD(&dma->channels);
215 	dma->dev = dev;
216 
217 	dma_cap_set(DMA_INTERRUPT, dma->cap_mask);
218 	dma_cap_set(DMA_PRIVATE, dma->cap_mask);
219 	dma_cap_set(DMA_COMPLETION_NO_ORDER, dma->cap_mask);
220 	dma->device_release = idxd_dma_release;
221 
222 	dma->device_prep_dma_interrupt = idxd_dma_prep_interrupt;
223 	if (idxd->hw.opcap.bits[0] & IDXD_OPCAP_MEMMOVE) {
224 		dma_cap_set(DMA_MEMCPY, dma->cap_mask);
225 		dma->device_prep_dma_memcpy = idxd_dma_submit_memcpy;
226 	}
227 
228 	dma->device_tx_status = idxd_dma_tx_status;
229 	dma->device_issue_pending = idxd_dma_issue_pending;
230 	dma->device_alloc_chan_resources = idxd_dma_alloc_chan_resources;
231 	dma->device_free_chan_resources = idxd_dma_free_chan_resources;
232 
233 	rc = dma_async_device_register(dma);
234 	if (rc < 0) {
235 		kfree(idxd_dma);
236 		return rc;
237 	}
238 
239 	idxd_dma->idxd = idxd;
240 	/*
241 	 * This pointer is protected by the refs taken by the dma_chan. It will remain valid
242 	 * as long as there are outstanding channels.
243 	 */
244 	idxd->idxd_dma = idxd_dma;
245 	return 0;
246 }
247 
idxd_unregister_dma_device(struct idxd_device * idxd)248 void idxd_unregister_dma_device(struct idxd_device *idxd)
249 {
250 	dma_async_device_unregister(&idxd->idxd_dma->dma);
251 }
252 
idxd_register_dma_channel(struct idxd_wq * wq)253 static int idxd_register_dma_channel(struct idxd_wq *wq)
254 {
255 	struct idxd_device *idxd = wq->idxd;
256 	struct dma_device *dma = &idxd->idxd_dma->dma;
257 	struct device *dev = &idxd->pdev->dev;
258 	struct idxd_dma_chan *idxd_chan;
259 	struct dma_chan *chan;
260 	int rc, i;
261 
262 	idxd_chan = kzalloc_node(sizeof(*idxd_chan), GFP_KERNEL, dev_to_node(dev));
263 	if (!idxd_chan)
264 		return -ENOMEM;
265 
266 	chan = &idxd_chan->chan;
267 	chan->device = dma;
268 	list_add_tail(&chan->device_node, &dma->channels);
269 
270 	for (i = 0; i < wq->num_descs; i++) {
271 		struct idxd_desc *desc = wq->descs[i];
272 
273 		dma_async_tx_descriptor_init(&desc->txd, chan);
274 		desc->txd.tx_submit = idxd_dma_tx_submit;
275 	}
276 
277 	rc = dma_async_device_channel_register(dma, chan);
278 	if (rc < 0) {
279 		kfree(idxd_chan);
280 		return rc;
281 	}
282 
283 	wq->idxd_chan = idxd_chan;
284 	idxd_chan->wq = wq;
285 	get_device(wq_confdev(wq));
286 
287 	return 0;
288 }
289 
idxd_unregister_dma_channel(struct idxd_wq * wq)290 static void idxd_unregister_dma_channel(struct idxd_wq *wq)
291 {
292 	struct idxd_dma_chan *idxd_chan = wq->idxd_chan;
293 	struct dma_chan *chan = &idxd_chan->chan;
294 	struct idxd_dma_dev *idxd_dma = wq->idxd->idxd_dma;
295 
296 	dma_async_device_channel_unregister(&idxd_dma->dma, chan);
297 	list_del(&chan->device_node);
298 	kfree(wq->idxd_chan);
299 	wq->idxd_chan = NULL;
300 	put_device(wq_confdev(wq));
301 }
302 
idxd_dmaengine_drv_probe(struct idxd_dev * idxd_dev)303 static int idxd_dmaengine_drv_probe(struct idxd_dev *idxd_dev)
304 {
305 	struct device *dev = &idxd_dev->conf_dev;
306 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
307 	struct idxd_device *idxd = wq->idxd;
308 	int rc;
309 
310 	if (idxd->state != IDXD_DEV_ENABLED)
311 		return -ENXIO;
312 
313 	mutex_lock(&wq->wq_lock);
314 	wq->type = IDXD_WQT_KERNEL;
315 
316 	rc = drv_enable_wq(wq);
317 	if (rc < 0) {
318 		dev_dbg(dev, "Enable wq %d failed: %d\n", wq->id, rc);
319 		rc = -ENXIO;
320 		goto err;
321 	}
322 
323 	rc = idxd_register_dma_channel(wq);
324 	if (rc < 0) {
325 		idxd->cmd_status = IDXD_SCMD_DMA_CHAN_ERR;
326 		dev_dbg(dev, "Failed to register dma channel\n");
327 		goto err_dma;
328 	}
329 
330 	idxd->cmd_status = 0;
331 	mutex_unlock(&wq->wq_lock);
332 	return 0;
333 
334 err_dma:
335 	drv_disable_wq(wq);
336 err:
337 	wq->type = IDXD_WQT_NONE;
338 	mutex_unlock(&wq->wq_lock);
339 	return rc;
340 }
341 
idxd_dmaengine_drv_remove(struct idxd_dev * idxd_dev)342 static void idxd_dmaengine_drv_remove(struct idxd_dev *idxd_dev)
343 {
344 	struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
345 
346 	mutex_lock(&wq->wq_lock);
347 	__idxd_wq_quiesce(wq);
348 	idxd_unregister_dma_channel(wq);
349 	drv_disable_wq(wq);
350 	mutex_unlock(&wq->wq_lock);
351 }
352 
353 static enum idxd_dev_type dev_types[] = {
354 	IDXD_DEV_WQ,
355 	IDXD_DEV_NONE,
356 };
357 
358 struct idxd_device_driver idxd_dmaengine_drv = {
359 	.probe = idxd_dmaengine_drv_probe,
360 	.remove = idxd_dmaengine_drv_remove,
361 	.name = "dmaengine",
362 	.type = dev_types,
363 };
364 EXPORT_SYMBOL_GPL(idxd_dmaengine_drv);
365