1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright 2017 IBM Corp.
3 #include <linux/sched/mm.h>
4 #include "trace.h"
5 #include "ocxl_internal.h"
6 
ocxl_context_alloc(struct ocxl_context ** context,struct ocxl_afu * afu,struct address_space * mapping)7 int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu,
8 		struct address_space *mapping)
9 {
10 	int pasid;
11 	struct ocxl_context *ctx;
12 
13 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
14 	if (!ctx)
15 		return -ENOMEM;
16 
17 	ctx->afu = afu;
18 	mutex_lock(&afu->contexts_lock);
19 	pasid = idr_alloc(&afu->contexts_idr, ctx, afu->pasid_base,
20 			afu->pasid_base + afu->pasid_max, GFP_KERNEL);
21 	if (pasid < 0) {
22 		mutex_unlock(&afu->contexts_lock);
23 		kfree(ctx);
24 		return pasid;
25 	}
26 	afu->pasid_count++;
27 	mutex_unlock(&afu->contexts_lock);
28 
29 	ctx->pasid = pasid;
30 	ctx->status = OPENED;
31 	mutex_init(&ctx->status_mutex);
32 	ctx->mapping = mapping;
33 	mutex_init(&ctx->mapping_lock);
34 	init_waitqueue_head(&ctx->events_wq);
35 	mutex_init(&ctx->xsl_error_lock);
36 	mutex_init(&ctx->irq_lock);
37 	idr_init(&ctx->irq_idr);
38 	ctx->tidr = 0;
39 
40 	/*
41 	 * Keep a reference on the AFU to make sure it's valid for the
42 	 * duration of the life of the context
43 	 */
44 	ocxl_afu_get(afu);
45 	*context = ctx;
46 	return 0;
47 }
48 EXPORT_SYMBOL_GPL(ocxl_context_alloc);
49 
50 /*
51  * Callback for when a translation fault triggers an error
52  * data:	a pointer to the context which triggered the fault
53  * addr:	the address that triggered the error
54  * dsisr:	the value of the PPC64 dsisr register
55  */
xsl_fault_error(void * data,u64 addr,u64 dsisr)56 static void xsl_fault_error(void *data, u64 addr, u64 dsisr)
57 {
58 	struct ocxl_context *ctx = (struct ocxl_context *) data;
59 
60 	mutex_lock(&ctx->xsl_error_lock);
61 	ctx->xsl_error.addr = addr;
62 	ctx->xsl_error.dsisr = dsisr;
63 	ctx->xsl_error.count++;
64 	mutex_unlock(&ctx->xsl_error_lock);
65 
66 	wake_up_all(&ctx->events_wq);
67 }
68 
ocxl_context_attach(struct ocxl_context * ctx,u64 amr,struct mm_struct * mm)69 int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm)
70 {
71 	int rc;
72 	unsigned long pidr = 0;
73 	struct pci_dev *dev;
74 
75 	// Locks both status & tidr
76 	mutex_lock(&ctx->status_mutex);
77 	if (ctx->status != OPENED) {
78 		rc = -EIO;
79 		goto out;
80 	}
81 
82 	if (mm)
83 		pidr = mm->context.id;
84 
85 	dev = to_pci_dev(ctx->afu->fn->dev.parent);
86 	rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, pidr, ctx->tidr,
87 			      amr, pci_dev_id(dev), mm, xsl_fault_error, ctx);
88 	if (rc)
89 		goto out;
90 
91 	ctx->status = ATTACHED;
92 out:
93 	mutex_unlock(&ctx->status_mutex);
94 	return rc;
95 }
96 EXPORT_SYMBOL_GPL(ocxl_context_attach);
97 
map_afu_irq(struct vm_area_struct * vma,unsigned long address,u64 offset,struct ocxl_context * ctx)98 static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address,
99 		u64 offset, struct ocxl_context *ctx)
100 {
101 	u64 trigger_addr;
102 	int irq_id = ocxl_irq_offset_to_id(ctx, offset);
103 
104 	trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id);
105 	if (!trigger_addr)
106 		return VM_FAULT_SIGBUS;
107 
108 	return vmf_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT);
109 }
110 
map_pp_mmio(struct vm_area_struct * vma,unsigned long address,u64 offset,struct ocxl_context * ctx)111 static vm_fault_t map_pp_mmio(struct vm_area_struct *vma, unsigned long address,
112 		u64 offset, struct ocxl_context *ctx)
113 {
114 	u64 pp_mmio_addr;
115 	int pasid_off;
116 	vm_fault_t ret;
117 
118 	if (offset >= ctx->afu->config.pp_mmio_stride)
119 		return VM_FAULT_SIGBUS;
120 
121 	mutex_lock(&ctx->status_mutex);
122 	if (ctx->status != ATTACHED) {
123 		mutex_unlock(&ctx->status_mutex);
124 		pr_debug("%s: Context not attached, failing mmio mmap\n",
125 			__func__);
126 		return VM_FAULT_SIGBUS;
127 	}
128 
129 	pasid_off = ctx->pasid - ctx->afu->pasid_base;
130 	pp_mmio_addr = ctx->afu->pp_mmio_start +
131 		pasid_off * ctx->afu->config.pp_mmio_stride +
132 		offset;
133 
134 	ret = vmf_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT);
135 	mutex_unlock(&ctx->status_mutex);
136 	return ret;
137 }
138 
ocxl_mmap_fault(struct vm_fault * vmf)139 static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf)
140 {
141 	struct vm_area_struct *vma = vmf->vma;
142 	struct ocxl_context *ctx = vma->vm_file->private_data;
143 	u64 offset;
144 	vm_fault_t ret;
145 
146 	offset = vmf->pgoff << PAGE_SHIFT;
147 	pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__,
148 		ctx->pasid, vmf->address, offset);
149 
150 	if (offset < ctx->afu->irq_base_offset)
151 		ret = map_pp_mmio(vma, vmf->address, offset, ctx);
152 	else
153 		ret = map_afu_irq(vma, vmf->address, offset, ctx);
154 	return ret;
155 }
156 
157 static const struct vm_operations_struct ocxl_vmops = {
158 	.fault = ocxl_mmap_fault,
159 };
160 
check_mmap_afu_irq(struct ocxl_context * ctx,struct vm_area_struct * vma)161 static int check_mmap_afu_irq(struct ocxl_context *ctx,
162 			struct vm_area_struct *vma)
163 {
164 	int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT);
165 
166 	/* only one page */
167 	if (vma_pages(vma) != 1)
168 		return -EINVAL;
169 
170 	/* check offset validty */
171 	if (!ocxl_afu_irq_get_addr(ctx, irq_id))
172 		return -EINVAL;
173 
174 	/*
175 	 * trigger page should only be accessible in write mode.
176 	 *
177 	 * It's a bit theoretical, as a page mmaped with only
178 	 * PROT_WRITE is currently readable, but it doesn't hurt.
179 	 */
180 	if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) ||
181 		!(vma->vm_flags & VM_WRITE))
182 		return -EINVAL;
183 	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
184 	return 0;
185 }
186 
check_mmap_mmio(struct ocxl_context * ctx,struct vm_area_struct * vma)187 static int check_mmap_mmio(struct ocxl_context *ctx,
188 			struct vm_area_struct *vma)
189 {
190 	if ((vma_pages(vma) + vma->vm_pgoff) >
191 		(ctx->afu->config.pp_mmio_stride >> PAGE_SHIFT))
192 		return -EINVAL;
193 	return 0;
194 }
195 
ocxl_context_mmap(struct ocxl_context * ctx,struct vm_area_struct * vma)196 int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma)
197 {
198 	int rc;
199 
200 	if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset)
201 		rc = check_mmap_mmio(ctx, vma);
202 	else
203 		rc = check_mmap_afu_irq(ctx, vma);
204 	if (rc)
205 		return rc;
206 
207 	vm_flags_set(vma, VM_IO | VM_PFNMAP);
208 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
209 	vma->vm_ops = &ocxl_vmops;
210 	return 0;
211 }
212 
ocxl_context_detach(struct ocxl_context * ctx)213 int ocxl_context_detach(struct ocxl_context *ctx)
214 {
215 	struct pci_dev *dev;
216 	int afu_control_pos;
217 	enum ocxl_context_status status;
218 	int rc;
219 
220 	mutex_lock(&ctx->status_mutex);
221 	status = ctx->status;
222 	ctx->status = CLOSED;
223 	mutex_unlock(&ctx->status_mutex);
224 	if (status != ATTACHED)
225 		return 0;
226 
227 	dev = to_pci_dev(ctx->afu->fn->dev.parent);
228 	afu_control_pos = ctx->afu->config.dvsec_afu_control_pos;
229 
230 	mutex_lock(&ctx->afu->afu_control_lock);
231 	rc = ocxl_config_terminate_pasid(dev, afu_control_pos, ctx->pasid);
232 	mutex_unlock(&ctx->afu->afu_control_lock);
233 	trace_ocxl_terminate_pasid(ctx->pasid, rc);
234 	if (rc) {
235 		/*
236 		 * If we timeout waiting for the AFU to terminate the
237 		 * pasid, then it's dangerous to clean up the Process
238 		 * Element entry in the SPA, as it may be referenced
239 		 * in the future by the AFU. In which case, we would
240 		 * checkstop because of an invalid PE access (FIR
241 		 * register 2, bit 42). So leave the PE
242 		 * defined. Caller shouldn't free the context so that
243 		 * PASID remains allocated.
244 		 *
245 		 * A link reset will be required to cleanup the AFU
246 		 * and the SPA.
247 		 */
248 		if (rc == -EBUSY)
249 			return rc;
250 	}
251 	rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid);
252 	if (rc) {
253 		dev_warn(&dev->dev,
254 			"Couldn't remove PE entry cleanly: %d\n", rc);
255 	}
256 	return 0;
257 }
258 EXPORT_SYMBOL_GPL(ocxl_context_detach);
259 
ocxl_context_detach_all(struct ocxl_afu * afu)260 void ocxl_context_detach_all(struct ocxl_afu *afu)
261 {
262 	struct ocxl_context *ctx;
263 	int tmp;
264 
265 	mutex_lock(&afu->contexts_lock);
266 	idr_for_each_entry(&afu->contexts_idr, ctx, tmp) {
267 		ocxl_context_detach(ctx);
268 		/*
269 		 * We are force detaching - remove any active mmio
270 		 * mappings so userspace cannot interfere with the
271 		 * card if it comes back.  Easiest way to exercise
272 		 * this is to unbind and rebind the driver via sysfs
273 		 * while it is in use.
274 		 */
275 		mutex_lock(&ctx->mapping_lock);
276 		if (ctx->mapping)
277 			unmap_mapping_range(ctx->mapping, 0, 0, 1);
278 		mutex_unlock(&ctx->mapping_lock);
279 	}
280 	mutex_unlock(&afu->contexts_lock);
281 }
282 
ocxl_context_free(struct ocxl_context * ctx)283 void ocxl_context_free(struct ocxl_context *ctx)
284 {
285 	mutex_lock(&ctx->afu->contexts_lock);
286 	ctx->afu->pasid_count--;
287 	idr_remove(&ctx->afu->contexts_idr, ctx->pasid);
288 	mutex_unlock(&ctx->afu->contexts_lock);
289 
290 	ocxl_afu_irq_free_all(ctx);
291 	idr_destroy(&ctx->irq_idr);
292 	/* reference to the AFU taken in ocxl_context_alloc() */
293 	ocxl_afu_put(ctx->afu);
294 	kfree(ctx);
295 }
296 EXPORT_SYMBOL_GPL(ocxl_context_free);
297