1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/platform_device.h>
4 #include <linux/memregion.h>
5 #include <linux/workqueue.h>
6 #include <linux/debugfs.h>
7 #include <linux/device.h>
8 #include <linux/module.h>
9 #include <linux/pci.h>
10 #include <linux/slab.h>
11 #include <linux/idr.h>
12 #include <cxlmem.h>
13 #include <cxlpci.h>
14 #include <cxl.h>
15 #include "core.h"
16 
17 /**
18  * DOC: cxl core
19  *
20  * The CXL core provides a set of interfaces that can be consumed by CXL aware
21  * drivers. The interfaces allow for creation, modification, and destruction of
22  * regions, memory devices, ports, and decoders. CXL aware drivers must register
23  * with the CXL core via these interfaces in order to be able to participate in
24  * cross-device interleave coordination. The CXL core also establishes and
25  * maintains the bridge to the nvdimm subsystem.
26  *
27  * CXL core introduces sysfs hierarchy to control the devices that are
28  * instantiated by the core.
29  */
30 
31 /*
32  * All changes to the interleave configuration occur with this lock held
33  * for write.
34  */
35 DECLARE_RWSEM(cxl_region_rwsem);
36 
37 static DEFINE_IDA(cxl_port_ida);
38 static DEFINE_XARRAY(cxl_root_buses);
39 
cxl_num_decoders_committed(struct cxl_port * port)40 int cxl_num_decoders_committed(struct cxl_port *port)
41 {
42 	lockdep_assert_held(&cxl_region_rwsem);
43 
44 	return port->commit_end + 1;
45 }
46 
devtype_show(struct device * dev,struct device_attribute * attr,char * buf)47 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
48 			    char *buf)
49 {
50 	return sysfs_emit(buf, "%s\n", dev->type->name);
51 }
52 static DEVICE_ATTR_RO(devtype);
53 
cxl_device_id(const struct device * dev)54 static int cxl_device_id(const struct device *dev)
55 {
56 	if (dev->type == &cxl_nvdimm_bridge_type)
57 		return CXL_DEVICE_NVDIMM_BRIDGE;
58 	if (dev->type == &cxl_nvdimm_type)
59 		return CXL_DEVICE_NVDIMM;
60 	if (dev->type == CXL_PMEM_REGION_TYPE())
61 		return CXL_DEVICE_PMEM_REGION;
62 	if (dev->type == CXL_DAX_REGION_TYPE())
63 		return CXL_DEVICE_DAX_REGION;
64 	if (is_cxl_port(dev)) {
65 		if (is_cxl_root(to_cxl_port(dev)))
66 			return CXL_DEVICE_ROOT;
67 		return CXL_DEVICE_PORT;
68 	}
69 	if (is_cxl_memdev(dev))
70 		return CXL_DEVICE_MEMORY_EXPANDER;
71 	if (dev->type == CXL_REGION_TYPE())
72 		return CXL_DEVICE_REGION;
73 	if (dev->type == &cxl_pmu_type)
74 		return CXL_DEVICE_PMU;
75 	return 0;
76 }
77 
modalias_show(struct device * dev,struct device_attribute * attr,char * buf)78 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
79 			     char *buf)
80 {
81 	return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
82 }
83 static DEVICE_ATTR_RO(modalias);
84 
85 static struct attribute *cxl_base_attributes[] = {
86 	&dev_attr_devtype.attr,
87 	&dev_attr_modalias.attr,
88 	NULL,
89 };
90 
91 struct attribute_group cxl_base_attribute_group = {
92 	.attrs = cxl_base_attributes,
93 };
94 
start_show(struct device * dev,struct device_attribute * attr,char * buf)95 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
96 			  char *buf)
97 {
98 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
99 
100 	return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
101 }
102 static DEVICE_ATTR_ADMIN_RO(start);
103 
size_show(struct device * dev,struct device_attribute * attr,char * buf)104 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
105 			char *buf)
106 {
107 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
108 
109 	return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
110 }
111 static DEVICE_ATTR_RO(size);
112 
113 #define CXL_DECODER_FLAG_ATTR(name, flag)                            \
114 static ssize_t name##_show(struct device *dev,                       \
115 			   struct device_attribute *attr, char *buf) \
116 {                                                                    \
117 	struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
118                                                                      \
119 	return sysfs_emit(buf, "%s\n",                               \
120 			  (cxld->flags & (flag)) ? "1" : "0");       \
121 }                                                                    \
122 static DEVICE_ATTR_RO(name)
123 
124 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
125 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
126 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
127 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
128 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
129 
target_type_show(struct device * dev,struct device_attribute * attr,char * buf)130 static ssize_t target_type_show(struct device *dev,
131 				struct device_attribute *attr, char *buf)
132 {
133 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
134 
135 	switch (cxld->target_type) {
136 	case CXL_DECODER_DEVMEM:
137 		return sysfs_emit(buf, "accelerator\n");
138 	case CXL_DECODER_HOSTONLYMEM:
139 		return sysfs_emit(buf, "expander\n");
140 	}
141 	return -ENXIO;
142 }
143 static DEVICE_ATTR_RO(target_type);
144 
emit_target_list(struct cxl_switch_decoder * cxlsd,char * buf)145 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
146 {
147 	struct cxl_decoder *cxld = &cxlsd->cxld;
148 	ssize_t offset = 0;
149 	int i, rc = 0;
150 
151 	for (i = 0; i < cxld->interleave_ways; i++) {
152 		struct cxl_dport *dport = cxlsd->target[i];
153 		struct cxl_dport *next = NULL;
154 
155 		if (!dport)
156 			break;
157 
158 		if (i + 1 < cxld->interleave_ways)
159 			next = cxlsd->target[i + 1];
160 		rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
161 				   next ? "," : "");
162 		if (rc < 0)
163 			return rc;
164 		offset += rc;
165 	}
166 
167 	return offset;
168 }
169 
target_list_show(struct device * dev,struct device_attribute * attr,char * buf)170 static ssize_t target_list_show(struct device *dev,
171 				struct device_attribute *attr, char *buf)
172 {
173 	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
174 	ssize_t offset;
175 	int rc;
176 
177 	guard(rwsem_read)(&cxl_region_rwsem);
178 	rc = emit_target_list(cxlsd, buf);
179 	if (rc < 0)
180 		return rc;
181 	offset = rc;
182 
183 	rc = sysfs_emit_at(buf, offset, "\n");
184 	if (rc < 0)
185 		return rc;
186 
187 	return offset + rc;
188 }
189 static DEVICE_ATTR_RO(target_list);
190 
mode_show(struct device * dev,struct device_attribute * attr,char * buf)191 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
192 			 char *buf)
193 {
194 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
195 
196 	return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxled->mode));
197 }
198 
mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)199 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
200 			  const char *buf, size_t len)
201 {
202 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
203 	enum cxl_decoder_mode mode;
204 	ssize_t rc;
205 
206 	if (sysfs_streq(buf, "pmem"))
207 		mode = CXL_DECODER_PMEM;
208 	else if (sysfs_streq(buf, "ram"))
209 		mode = CXL_DECODER_RAM;
210 	else
211 		return -EINVAL;
212 
213 	rc = cxl_dpa_set_mode(cxled, mode);
214 	if (rc)
215 		return rc;
216 
217 	return len;
218 }
219 static DEVICE_ATTR_RW(mode);
220 
dpa_resource_show(struct device * dev,struct device_attribute * attr,char * buf)221 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
222 			    char *buf)
223 {
224 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
225 
226 	guard(rwsem_read)(&cxl_dpa_rwsem);
227 	return sysfs_emit(buf, "%#llx\n", (u64)cxl_dpa_resource_start(cxled));
228 }
229 static DEVICE_ATTR_RO(dpa_resource);
230 
dpa_size_show(struct device * dev,struct device_attribute * attr,char * buf)231 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
232 			     char *buf)
233 {
234 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
235 	resource_size_t size = cxl_dpa_size(cxled);
236 
237 	return sysfs_emit(buf, "%pa\n", &size);
238 }
239 
dpa_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)240 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
241 			      const char *buf, size_t len)
242 {
243 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
244 	unsigned long long size;
245 	ssize_t rc;
246 
247 	rc = kstrtoull(buf, 0, &size);
248 	if (rc)
249 		return rc;
250 
251 	if (!IS_ALIGNED(size, SZ_256M))
252 		return -EINVAL;
253 
254 	rc = cxl_dpa_free(cxled);
255 	if (rc)
256 		return rc;
257 
258 	if (size == 0)
259 		return len;
260 
261 	rc = cxl_dpa_alloc(cxled, size);
262 	if (rc)
263 		return rc;
264 
265 	return len;
266 }
267 static DEVICE_ATTR_RW(dpa_size);
268 
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)269 static ssize_t interleave_granularity_show(struct device *dev,
270 					   struct device_attribute *attr,
271 					   char *buf)
272 {
273 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
274 
275 	return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
276 }
277 
278 static DEVICE_ATTR_RO(interleave_granularity);
279 
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)280 static ssize_t interleave_ways_show(struct device *dev,
281 				    struct device_attribute *attr, char *buf)
282 {
283 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
284 
285 	return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
286 }
287 
288 static DEVICE_ATTR_RO(interleave_ways);
289 
290 static struct attribute *cxl_decoder_base_attrs[] = {
291 	&dev_attr_start.attr,
292 	&dev_attr_size.attr,
293 	&dev_attr_locked.attr,
294 	&dev_attr_interleave_granularity.attr,
295 	&dev_attr_interleave_ways.attr,
296 	NULL,
297 };
298 
299 static struct attribute_group cxl_decoder_base_attribute_group = {
300 	.attrs = cxl_decoder_base_attrs,
301 };
302 
303 static struct attribute *cxl_decoder_root_attrs[] = {
304 	&dev_attr_cap_pmem.attr,
305 	&dev_attr_cap_ram.attr,
306 	&dev_attr_cap_type2.attr,
307 	&dev_attr_cap_type3.attr,
308 	&dev_attr_target_list.attr,
309 	SET_CXL_REGION_ATTR(create_pmem_region)
310 	SET_CXL_REGION_ATTR(create_ram_region)
311 	SET_CXL_REGION_ATTR(delete_region)
312 	NULL,
313 };
314 
can_create_pmem(struct cxl_root_decoder * cxlrd)315 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
316 {
317 	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
318 
319 	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
320 }
321 
can_create_ram(struct cxl_root_decoder * cxlrd)322 static bool can_create_ram(struct cxl_root_decoder *cxlrd)
323 {
324 	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_RAM;
325 
326 	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
327 }
328 
cxl_root_decoder_visible(struct kobject * kobj,struct attribute * a,int n)329 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
330 {
331 	struct device *dev = kobj_to_dev(kobj);
332 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
333 
334 	if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
335 		return 0;
336 
337 	if (a == CXL_REGION_ATTR(create_ram_region) && !can_create_ram(cxlrd))
338 		return 0;
339 
340 	if (a == CXL_REGION_ATTR(delete_region) &&
341 	    !(can_create_pmem(cxlrd) || can_create_ram(cxlrd)))
342 		return 0;
343 
344 	return a->mode;
345 }
346 
347 static struct attribute_group cxl_decoder_root_attribute_group = {
348 	.attrs = cxl_decoder_root_attrs,
349 	.is_visible = cxl_root_decoder_visible,
350 };
351 
352 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
353 	&cxl_decoder_root_attribute_group,
354 	&cxl_decoder_base_attribute_group,
355 	&cxl_base_attribute_group,
356 	NULL,
357 };
358 
359 static struct attribute *cxl_decoder_switch_attrs[] = {
360 	&dev_attr_target_type.attr,
361 	&dev_attr_target_list.attr,
362 	SET_CXL_REGION_ATTR(region)
363 	NULL,
364 };
365 
366 static struct attribute_group cxl_decoder_switch_attribute_group = {
367 	.attrs = cxl_decoder_switch_attrs,
368 };
369 
370 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
371 	&cxl_decoder_switch_attribute_group,
372 	&cxl_decoder_base_attribute_group,
373 	&cxl_base_attribute_group,
374 	NULL,
375 };
376 
377 static struct attribute *cxl_decoder_endpoint_attrs[] = {
378 	&dev_attr_target_type.attr,
379 	&dev_attr_mode.attr,
380 	&dev_attr_dpa_size.attr,
381 	&dev_attr_dpa_resource.attr,
382 	SET_CXL_REGION_ATTR(region)
383 	NULL,
384 };
385 
386 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
387 	.attrs = cxl_decoder_endpoint_attrs,
388 };
389 
390 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
391 	&cxl_decoder_base_attribute_group,
392 	&cxl_decoder_endpoint_attribute_group,
393 	&cxl_base_attribute_group,
394 	NULL,
395 };
396 
__cxl_decoder_release(struct cxl_decoder * cxld)397 static void __cxl_decoder_release(struct cxl_decoder *cxld)
398 {
399 	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
400 
401 	ida_free(&port->decoder_ida, cxld->id);
402 	put_device(&port->dev);
403 }
404 
cxl_endpoint_decoder_release(struct device * dev)405 static void cxl_endpoint_decoder_release(struct device *dev)
406 {
407 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
408 
409 	__cxl_decoder_release(&cxled->cxld);
410 	kfree(cxled);
411 }
412 
cxl_switch_decoder_release(struct device * dev)413 static void cxl_switch_decoder_release(struct device *dev)
414 {
415 	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
416 
417 	__cxl_decoder_release(&cxlsd->cxld);
418 	kfree(cxlsd);
419 }
420 
to_cxl_root_decoder(struct device * dev)421 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
422 {
423 	if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
424 			  "not a cxl_root_decoder device\n"))
425 		return NULL;
426 	return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
427 }
428 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
429 
cxl_root_decoder_release(struct device * dev)430 static void cxl_root_decoder_release(struct device *dev)
431 {
432 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
433 
434 	if (atomic_read(&cxlrd->region_id) >= 0)
435 		memregion_free(atomic_read(&cxlrd->region_id));
436 	__cxl_decoder_release(&cxlrd->cxlsd.cxld);
437 	kfree(cxlrd);
438 }
439 
440 static const struct device_type cxl_decoder_endpoint_type = {
441 	.name = "cxl_decoder_endpoint",
442 	.release = cxl_endpoint_decoder_release,
443 	.groups = cxl_decoder_endpoint_attribute_groups,
444 };
445 
446 static const struct device_type cxl_decoder_switch_type = {
447 	.name = "cxl_decoder_switch",
448 	.release = cxl_switch_decoder_release,
449 	.groups = cxl_decoder_switch_attribute_groups,
450 };
451 
452 static const struct device_type cxl_decoder_root_type = {
453 	.name = "cxl_decoder_root",
454 	.release = cxl_root_decoder_release,
455 	.groups = cxl_decoder_root_attribute_groups,
456 };
457 
is_endpoint_decoder(struct device * dev)458 bool is_endpoint_decoder(struct device *dev)
459 {
460 	return dev->type == &cxl_decoder_endpoint_type;
461 }
462 EXPORT_SYMBOL_NS_GPL(is_endpoint_decoder, CXL);
463 
is_root_decoder(struct device * dev)464 bool is_root_decoder(struct device *dev)
465 {
466 	return dev->type == &cxl_decoder_root_type;
467 }
468 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
469 
is_switch_decoder(struct device * dev)470 bool is_switch_decoder(struct device *dev)
471 {
472 	return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
473 }
474 EXPORT_SYMBOL_NS_GPL(is_switch_decoder, CXL);
475 
to_cxl_decoder(struct device * dev)476 struct cxl_decoder *to_cxl_decoder(struct device *dev)
477 {
478 	if (dev_WARN_ONCE(dev,
479 			  !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
480 			  "not a cxl_decoder device\n"))
481 		return NULL;
482 	return container_of(dev, struct cxl_decoder, dev);
483 }
484 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
485 
to_cxl_endpoint_decoder(struct device * dev)486 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
487 {
488 	if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
489 			  "not a cxl_endpoint_decoder device\n"))
490 		return NULL;
491 	return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
492 }
493 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
494 
to_cxl_switch_decoder(struct device * dev)495 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
496 {
497 	if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
498 			  "not a cxl_switch_decoder device\n"))
499 		return NULL;
500 	return container_of(dev, struct cxl_switch_decoder, cxld.dev);
501 }
502 EXPORT_SYMBOL_NS_GPL(to_cxl_switch_decoder, CXL);
503 
cxl_ep_release(struct cxl_ep * ep)504 static void cxl_ep_release(struct cxl_ep *ep)
505 {
506 	put_device(ep->ep);
507 	kfree(ep);
508 }
509 
cxl_ep_remove(struct cxl_port * port,struct cxl_ep * ep)510 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
511 {
512 	if (!ep)
513 		return;
514 	xa_erase(&port->endpoints, (unsigned long) ep->ep);
515 	cxl_ep_release(ep);
516 }
517 
cxl_port_release(struct device * dev)518 static void cxl_port_release(struct device *dev)
519 {
520 	struct cxl_port *port = to_cxl_port(dev);
521 	unsigned long index;
522 	struct cxl_ep *ep;
523 
524 	xa_for_each(&port->endpoints, index, ep)
525 		cxl_ep_remove(port, ep);
526 	xa_destroy(&port->endpoints);
527 	xa_destroy(&port->dports);
528 	xa_destroy(&port->regions);
529 	ida_free(&cxl_port_ida, port->id);
530 	kfree(port);
531 }
532 
533 static const struct attribute_group *cxl_port_attribute_groups[] = {
534 	&cxl_base_attribute_group,
535 	NULL,
536 };
537 
538 static const struct device_type cxl_port_type = {
539 	.name = "cxl_port",
540 	.release = cxl_port_release,
541 	.groups = cxl_port_attribute_groups,
542 };
543 
is_cxl_port(const struct device * dev)544 bool is_cxl_port(const struct device *dev)
545 {
546 	return dev->type == &cxl_port_type;
547 }
548 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
549 
to_cxl_port(const struct device * dev)550 struct cxl_port *to_cxl_port(const struct device *dev)
551 {
552 	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
553 			  "not a cxl_port device\n"))
554 		return NULL;
555 	return container_of(dev, struct cxl_port, dev);
556 }
557 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
558 
unregister_port(void * _port)559 static void unregister_port(void *_port)
560 {
561 	struct cxl_port *port = _port;
562 	struct cxl_port *parent;
563 	struct device *lock_dev;
564 
565 	if (is_cxl_root(port))
566 		parent = NULL;
567 	else
568 		parent = to_cxl_port(port->dev.parent);
569 
570 	/*
571 	 * CXL root port's and the first level of ports are unregistered
572 	 * under the platform firmware device lock, all other ports are
573 	 * unregistered while holding their parent port lock.
574 	 */
575 	if (!parent)
576 		lock_dev = port->uport_dev;
577 	else if (is_cxl_root(parent))
578 		lock_dev = parent->uport_dev;
579 	else
580 		lock_dev = &parent->dev;
581 
582 	device_lock_assert(lock_dev);
583 	port->dead = true;
584 	device_unregister(&port->dev);
585 }
586 
cxl_unlink_uport(void * _port)587 static void cxl_unlink_uport(void *_port)
588 {
589 	struct cxl_port *port = _port;
590 
591 	sysfs_remove_link(&port->dev.kobj, "uport");
592 }
593 
devm_cxl_link_uport(struct device * host,struct cxl_port * port)594 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
595 {
596 	int rc;
597 
598 	rc = sysfs_create_link(&port->dev.kobj, &port->uport_dev->kobj,
599 			       "uport");
600 	if (rc)
601 		return rc;
602 	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
603 }
604 
cxl_unlink_parent_dport(void * _port)605 static void cxl_unlink_parent_dport(void *_port)
606 {
607 	struct cxl_port *port = _port;
608 
609 	sysfs_remove_link(&port->dev.kobj, "parent_dport");
610 }
611 
devm_cxl_link_parent_dport(struct device * host,struct cxl_port * port,struct cxl_dport * parent_dport)612 static int devm_cxl_link_parent_dport(struct device *host,
613 				      struct cxl_port *port,
614 				      struct cxl_dport *parent_dport)
615 {
616 	int rc;
617 
618 	if (!parent_dport)
619 		return 0;
620 
621 	rc = sysfs_create_link(&port->dev.kobj, &parent_dport->dport_dev->kobj,
622 			       "parent_dport");
623 	if (rc)
624 		return rc;
625 	return devm_add_action_or_reset(host, cxl_unlink_parent_dport, port);
626 }
627 
628 static struct lock_class_key cxl_port_key;
629 
cxl_port_alloc(struct device * uport_dev,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)630 static struct cxl_port *cxl_port_alloc(struct device *uport_dev,
631 				       resource_size_t component_reg_phys,
632 				       struct cxl_dport *parent_dport)
633 {
634 	struct cxl_port *port;
635 	struct device *dev;
636 	int rc;
637 
638 	port = kzalloc(sizeof(*port), GFP_KERNEL);
639 	if (!port)
640 		return ERR_PTR(-ENOMEM);
641 
642 	rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
643 	if (rc < 0)
644 		goto err;
645 	port->id = rc;
646 	port->uport_dev = uport_dev;
647 
648 	/*
649 	 * The top-level cxl_port "cxl_root" does not have a cxl_port as
650 	 * its parent and it does not have any corresponding component
651 	 * registers as its decode is described by a fixed platform
652 	 * description.
653 	 */
654 	dev = &port->dev;
655 	if (parent_dport) {
656 		struct cxl_port *parent_port = parent_dport->port;
657 		struct cxl_port *iter;
658 
659 		dev->parent = &parent_port->dev;
660 		port->depth = parent_port->depth + 1;
661 		port->parent_dport = parent_dport;
662 
663 		/*
664 		 * walk to the host bridge, or the first ancestor that knows
665 		 * the host bridge
666 		 */
667 		iter = port;
668 		while (!iter->host_bridge &&
669 		       !is_cxl_root(to_cxl_port(iter->dev.parent)))
670 			iter = to_cxl_port(iter->dev.parent);
671 		if (iter->host_bridge)
672 			port->host_bridge = iter->host_bridge;
673 		else if (parent_dport->rch)
674 			port->host_bridge = parent_dport->dport_dev;
675 		else
676 			port->host_bridge = iter->uport_dev;
677 		dev_dbg(uport_dev, "host-bridge: %s\n",
678 			dev_name(port->host_bridge));
679 	} else
680 		dev->parent = uport_dev;
681 
682 	port->component_reg_phys = component_reg_phys;
683 	ida_init(&port->decoder_ida);
684 	port->hdm_end = -1;
685 	port->commit_end = -1;
686 	xa_init(&port->dports);
687 	xa_init(&port->endpoints);
688 	xa_init(&port->regions);
689 
690 	device_initialize(dev);
691 	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
692 	device_set_pm_not_required(dev);
693 	dev->bus = &cxl_bus_type;
694 	dev->type = &cxl_port_type;
695 
696 	return port;
697 
698 err:
699 	kfree(port);
700 	return ERR_PTR(rc);
701 }
702 
cxl_setup_comp_regs(struct device * host,struct cxl_register_map * map,resource_size_t component_reg_phys)703 static int cxl_setup_comp_regs(struct device *host, struct cxl_register_map *map,
704 			       resource_size_t component_reg_phys)
705 {
706 	if (component_reg_phys == CXL_RESOURCE_NONE)
707 		return 0;
708 
709 	*map = (struct cxl_register_map) {
710 		.host = host,
711 		.reg_type = CXL_REGLOC_RBI_COMPONENT,
712 		.resource = component_reg_phys,
713 		.max_size = CXL_COMPONENT_REG_BLOCK_SIZE,
714 	};
715 
716 	return cxl_setup_regs(map);
717 }
718 
cxl_port_setup_regs(struct cxl_port * port,resource_size_t component_reg_phys)719 static int cxl_port_setup_regs(struct cxl_port *port,
720 			resource_size_t component_reg_phys)
721 {
722 	if (dev_is_platform(port->uport_dev))
723 		return 0;
724 	return cxl_setup_comp_regs(&port->dev, &port->comp_map,
725 				   component_reg_phys);
726 }
727 
cxl_dport_setup_regs(struct device * host,struct cxl_dport * dport,resource_size_t component_reg_phys)728 static int cxl_dport_setup_regs(struct device *host, struct cxl_dport *dport,
729 				resource_size_t component_reg_phys)
730 {
731 	int rc;
732 
733 	if (dev_is_platform(dport->dport_dev))
734 		return 0;
735 
736 	/*
737 	 * use @dport->dport_dev for the context for error messages during
738 	 * register probing, and fixup @host after the fact, since @host may be
739 	 * NULL.
740 	 */
741 	rc = cxl_setup_comp_regs(dport->dport_dev, &dport->comp_map,
742 				 component_reg_phys);
743 	dport->comp_map.host = host;
744 	return rc;
745 }
746 
__devm_cxl_add_port(struct device * host,struct device * uport_dev,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)747 static struct cxl_port *__devm_cxl_add_port(struct device *host,
748 					    struct device *uport_dev,
749 					    resource_size_t component_reg_phys,
750 					    struct cxl_dport *parent_dport)
751 {
752 	struct cxl_port *port;
753 	struct device *dev;
754 	int rc;
755 
756 	port = cxl_port_alloc(uport_dev, component_reg_phys, parent_dport);
757 	if (IS_ERR(port))
758 		return port;
759 
760 	dev = &port->dev;
761 	if (is_cxl_memdev(uport_dev))
762 		rc = dev_set_name(dev, "endpoint%d", port->id);
763 	else if (parent_dport)
764 		rc = dev_set_name(dev, "port%d", port->id);
765 	else
766 		rc = dev_set_name(dev, "root%d", port->id);
767 	if (rc)
768 		goto err;
769 
770 	rc = cxl_port_setup_regs(port, component_reg_phys);
771 	if (rc)
772 		goto err;
773 
774 	rc = device_add(dev);
775 	if (rc)
776 		goto err;
777 
778 	rc = devm_add_action_or_reset(host, unregister_port, port);
779 	if (rc)
780 		return ERR_PTR(rc);
781 
782 	rc = devm_cxl_link_uport(host, port);
783 	if (rc)
784 		return ERR_PTR(rc);
785 
786 	rc = devm_cxl_link_parent_dport(host, port, parent_dport);
787 	if (rc)
788 		return ERR_PTR(rc);
789 
790 	return port;
791 
792 err:
793 	put_device(dev);
794 	return ERR_PTR(rc);
795 }
796 
797 /**
798  * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
799  * @host: host device for devm operations
800  * @uport_dev: "physical" device implementing this upstream port
801  * @component_reg_phys: (optional) for configurable cxl_port instances
802  * @parent_dport: next hop up in the CXL memory decode hierarchy
803  */
devm_cxl_add_port(struct device * host,struct device * uport_dev,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)804 struct cxl_port *devm_cxl_add_port(struct device *host,
805 				   struct device *uport_dev,
806 				   resource_size_t component_reg_phys,
807 				   struct cxl_dport *parent_dport)
808 {
809 	struct cxl_port *port, *parent_port;
810 
811 	port = __devm_cxl_add_port(host, uport_dev, component_reg_phys,
812 				   parent_dport);
813 
814 	parent_port = parent_dport ? parent_dport->port : NULL;
815 	if (IS_ERR(port)) {
816 		dev_dbg(uport_dev, "Failed to add%s%s%s: %ld\n",
817 			parent_port ? " port to " : "",
818 			parent_port ? dev_name(&parent_port->dev) : "",
819 			parent_port ? "" : " root port",
820 			PTR_ERR(port));
821 	} else {
822 		dev_dbg(uport_dev, "%s added%s%s%s\n",
823 			dev_name(&port->dev),
824 			parent_port ? " to " : "",
825 			parent_port ? dev_name(&parent_port->dev) : "",
826 			parent_port ? "" : " (root port)");
827 	}
828 
829 	return port;
830 }
831 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
832 
cxl_port_to_pci_bus(struct cxl_port * port)833 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
834 {
835 	/* There is no pci_bus associated with a CXL platform-root port */
836 	if (is_cxl_root(port))
837 		return NULL;
838 
839 	if (dev_is_pci(port->uport_dev)) {
840 		struct pci_dev *pdev = to_pci_dev(port->uport_dev);
841 
842 		return pdev->subordinate;
843 	}
844 
845 	return xa_load(&cxl_root_buses, (unsigned long)port->uport_dev);
846 }
847 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
848 
unregister_pci_bus(void * uport_dev)849 static void unregister_pci_bus(void *uport_dev)
850 {
851 	xa_erase(&cxl_root_buses, (unsigned long)uport_dev);
852 }
853 
devm_cxl_register_pci_bus(struct device * host,struct device * uport_dev,struct pci_bus * bus)854 int devm_cxl_register_pci_bus(struct device *host, struct device *uport_dev,
855 			      struct pci_bus *bus)
856 {
857 	int rc;
858 
859 	if (dev_is_pci(uport_dev))
860 		return -EINVAL;
861 
862 	rc = xa_insert(&cxl_root_buses, (unsigned long)uport_dev, bus,
863 		       GFP_KERNEL);
864 	if (rc)
865 		return rc;
866 	return devm_add_action_or_reset(host, unregister_pci_bus, uport_dev);
867 }
868 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
869 
dev_is_cxl_root_child(struct device * dev)870 static bool dev_is_cxl_root_child(struct device *dev)
871 {
872 	struct cxl_port *port, *parent;
873 
874 	if (!is_cxl_port(dev))
875 		return false;
876 
877 	port = to_cxl_port(dev);
878 	if (is_cxl_root(port))
879 		return false;
880 
881 	parent = to_cxl_port(port->dev.parent);
882 	if (is_cxl_root(parent))
883 		return true;
884 
885 	return false;
886 }
887 
find_cxl_root(struct cxl_port * port)888 struct cxl_port *find_cxl_root(struct cxl_port *port)
889 {
890 	struct cxl_port *iter = port;
891 
892 	while (iter && !is_cxl_root(iter))
893 		iter = to_cxl_port(iter->dev.parent);
894 
895 	if (!iter)
896 		return NULL;
897 	get_device(&iter->dev);
898 	return iter;
899 }
900 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
901 
find_dport(struct cxl_port * port,int id)902 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
903 {
904 	struct cxl_dport *dport;
905 	unsigned long index;
906 
907 	device_lock_assert(&port->dev);
908 	xa_for_each(&port->dports, index, dport)
909 		if (dport->port_id == id)
910 			return dport;
911 	return NULL;
912 }
913 
add_dport(struct cxl_port * port,struct cxl_dport * dport)914 static int add_dport(struct cxl_port *port, struct cxl_dport *dport)
915 {
916 	struct cxl_dport *dup;
917 	int rc;
918 
919 	device_lock_assert(&port->dev);
920 	dup = find_dport(port, dport->port_id);
921 	if (dup) {
922 		dev_err(&port->dev,
923 			"unable to add dport%d-%s non-unique port id (%s)\n",
924 			dport->port_id, dev_name(dport->dport_dev),
925 			dev_name(dup->dport_dev));
926 		return -EBUSY;
927 	}
928 
929 	rc = xa_insert(&port->dports, (unsigned long)dport->dport_dev, dport,
930 		       GFP_KERNEL);
931 	if (rc)
932 		return rc;
933 
934 	port->nr_dports++;
935 	return 0;
936 }
937 
938 /*
939  * Since root-level CXL dports cannot be enumerated by PCI they are not
940  * enumerated by the common port driver that acquires the port lock over
941  * dport add/remove. Instead, root dports are manually added by a
942  * platform driver and cond_cxl_root_lock() is used to take the missing
943  * port lock in that case.
944  */
cond_cxl_root_lock(struct cxl_port * port)945 static void cond_cxl_root_lock(struct cxl_port *port)
946 {
947 	if (is_cxl_root(port))
948 		device_lock(&port->dev);
949 }
950 
cond_cxl_root_unlock(struct cxl_port * port)951 static void cond_cxl_root_unlock(struct cxl_port *port)
952 {
953 	if (is_cxl_root(port))
954 		device_unlock(&port->dev);
955 }
956 
cxl_dport_remove(void * data)957 static void cxl_dport_remove(void *data)
958 {
959 	struct cxl_dport *dport = data;
960 	struct cxl_port *port = dport->port;
961 
962 	xa_erase(&port->dports, (unsigned long) dport->dport_dev);
963 	put_device(dport->dport_dev);
964 }
965 
cxl_dport_unlink(void * data)966 static void cxl_dport_unlink(void *data)
967 {
968 	struct cxl_dport *dport = data;
969 	struct cxl_port *port = dport->port;
970 	char link_name[CXL_TARGET_STRLEN];
971 
972 	sprintf(link_name, "dport%d", dport->port_id);
973 	sysfs_remove_link(&port->dev.kobj, link_name);
974 }
975 
976 static struct cxl_dport *
__devm_cxl_add_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t component_reg_phys,resource_size_t rcrb)977 __devm_cxl_add_dport(struct cxl_port *port, struct device *dport_dev,
978 		     int port_id, resource_size_t component_reg_phys,
979 		     resource_size_t rcrb)
980 {
981 	char link_name[CXL_TARGET_STRLEN];
982 	struct cxl_dport *dport;
983 	struct device *host;
984 	int rc;
985 
986 	if (is_cxl_root(port))
987 		host = port->uport_dev;
988 	else
989 		host = &port->dev;
990 
991 	if (!host->driver) {
992 		dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
993 			      dev_name(dport_dev));
994 		return ERR_PTR(-ENXIO);
995 	}
996 
997 	if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
998 	    CXL_TARGET_STRLEN)
999 		return ERR_PTR(-EINVAL);
1000 
1001 	dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
1002 	if (!dport)
1003 		return ERR_PTR(-ENOMEM);
1004 
1005 	dport->dport_dev = dport_dev;
1006 	dport->port_id = port_id;
1007 	dport->port = port;
1008 
1009 	if (rcrb == CXL_RESOURCE_NONE) {
1010 		rc = cxl_dport_setup_regs(&port->dev, dport,
1011 					  component_reg_phys);
1012 		if (rc)
1013 			return ERR_PTR(rc);
1014 	} else {
1015 		dport->rcrb.base = rcrb;
1016 		component_reg_phys = __rcrb_to_component(dport_dev, &dport->rcrb,
1017 							 CXL_RCRB_DOWNSTREAM);
1018 		if (component_reg_phys == CXL_RESOURCE_NONE) {
1019 			dev_warn(dport_dev, "Invalid Component Registers in RCRB");
1020 			return ERR_PTR(-ENXIO);
1021 		}
1022 
1023 		/*
1024 		 * RCH @dport is not ready to map until associated with its
1025 		 * memdev
1026 		 */
1027 		rc = cxl_dport_setup_regs(NULL, dport, component_reg_phys);
1028 		if (rc)
1029 			return ERR_PTR(rc);
1030 
1031 		dport->rch = true;
1032 	}
1033 
1034 	if (component_reg_phys != CXL_RESOURCE_NONE)
1035 		dev_dbg(dport_dev, "Component Registers found for dport: %pa\n",
1036 			&component_reg_phys);
1037 
1038 	cond_cxl_root_lock(port);
1039 	rc = add_dport(port, dport);
1040 	cond_cxl_root_unlock(port);
1041 	if (rc)
1042 		return ERR_PTR(rc);
1043 
1044 	get_device(dport_dev);
1045 	rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
1046 	if (rc)
1047 		return ERR_PTR(rc);
1048 
1049 	rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
1050 	if (rc)
1051 		return ERR_PTR(rc);
1052 
1053 	rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
1054 	if (rc)
1055 		return ERR_PTR(rc);
1056 
1057 	return dport;
1058 }
1059 
1060 /**
1061  * devm_cxl_add_dport - append VH downstream port data to a cxl_port
1062  * @port: the cxl_port that references this dport
1063  * @dport_dev: firmware or PCI device representing the dport
1064  * @port_id: identifier for this dport in a decoder's target list
1065  * @component_reg_phys: optional location of CXL component registers
1066  *
1067  * Note that dports are appended to the devm release action's of the
1068  * either the port's host (for root ports), or the port itself (for
1069  * switch ports)
1070  */
devm_cxl_add_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t component_reg_phys)1071 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
1072 				     struct device *dport_dev, int port_id,
1073 				     resource_size_t component_reg_phys)
1074 {
1075 	struct cxl_dport *dport;
1076 
1077 	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1078 				     component_reg_phys, CXL_RESOURCE_NONE);
1079 	if (IS_ERR(dport)) {
1080 		dev_dbg(dport_dev, "failed to add dport to %s: %ld\n",
1081 			dev_name(&port->dev), PTR_ERR(dport));
1082 	} else {
1083 		dev_dbg(dport_dev, "dport added to %s\n",
1084 			dev_name(&port->dev));
1085 	}
1086 
1087 	return dport;
1088 }
1089 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
1090 
1091 /**
1092  * devm_cxl_add_rch_dport - append RCH downstream port data to a cxl_port
1093  * @port: the cxl_port that references this dport
1094  * @dport_dev: firmware or PCI device representing the dport
1095  * @port_id: identifier for this dport in a decoder's target list
1096  * @rcrb: mandatory location of a Root Complex Register Block
1097  *
1098  * See CXL 3.0 9.11.8 CXL Devices Attached to an RCH
1099  */
devm_cxl_add_rch_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t rcrb)1100 struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
1101 					 struct device *dport_dev, int port_id,
1102 					 resource_size_t rcrb)
1103 {
1104 	struct cxl_dport *dport;
1105 
1106 	if (rcrb == CXL_RESOURCE_NONE) {
1107 		dev_dbg(&port->dev, "failed to add RCH dport, missing RCRB\n");
1108 		return ERR_PTR(-EINVAL);
1109 	}
1110 
1111 	dport = __devm_cxl_add_dport(port, dport_dev, port_id,
1112 				     CXL_RESOURCE_NONE, rcrb);
1113 	if (IS_ERR(dport)) {
1114 		dev_dbg(dport_dev, "failed to add RCH dport to %s: %ld\n",
1115 			dev_name(&port->dev), PTR_ERR(dport));
1116 	} else {
1117 		dev_dbg(dport_dev, "RCH dport added to %s\n",
1118 			dev_name(&port->dev));
1119 	}
1120 
1121 	return dport;
1122 }
1123 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_rch_dport, CXL);
1124 
add_ep(struct cxl_ep * new)1125 static int add_ep(struct cxl_ep *new)
1126 {
1127 	struct cxl_port *port = new->dport->port;
1128 	int rc;
1129 
1130 	device_lock(&port->dev);
1131 	if (port->dead) {
1132 		device_unlock(&port->dev);
1133 		return -ENXIO;
1134 	}
1135 	rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
1136 		       GFP_KERNEL);
1137 	device_unlock(&port->dev);
1138 
1139 	return rc;
1140 }
1141 
1142 /**
1143  * cxl_add_ep - register an endpoint's interest in a port
1144  * @dport: the dport that routes to @ep_dev
1145  * @ep_dev: device representing the endpoint
1146  *
1147  * Intermediate CXL ports are scanned based on the arrival of endpoints.
1148  * When those endpoints depart the port can be destroyed once all
1149  * endpoints that care about that port have been removed.
1150  */
cxl_add_ep(struct cxl_dport * dport,struct device * ep_dev)1151 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
1152 {
1153 	struct cxl_ep *ep;
1154 	int rc;
1155 
1156 	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
1157 	if (!ep)
1158 		return -ENOMEM;
1159 
1160 	ep->ep = get_device(ep_dev);
1161 	ep->dport = dport;
1162 
1163 	rc = add_ep(ep);
1164 	if (rc)
1165 		cxl_ep_release(ep);
1166 	return rc;
1167 }
1168 
1169 struct cxl_find_port_ctx {
1170 	const struct device *dport_dev;
1171 	const struct cxl_port *parent_port;
1172 	struct cxl_dport **dport;
1173 };
1174 
match_port_by_dport(struct device * dev,const void * data)1175 static int match_port_by_dport(struct device *dev, const void *data)
1176 {
1177 	const struct cxl_find_port_ctx *ctx = data;
1178 	struct cxl_dport *dport;
1179 	struct cxl_port *port;
1180 
1181 	if (!is_cxl_port(dev))
1182 		return 0;
1183 	if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
1184 		return 0;
1185 
1186 	port = to_cxl_port(dev);
1187 	dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1188 	if (ctx->dport)
1189 		*ctx->dport = dport;
1190 	return dport != NULL;
1191 }
1192 
__find_cxl_port(struct cxl_find_port_ctx * ctx)1193 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1194 {
1195 	struct device *dev;
1196 
1197 	if (!ctx->dport_dev)
1198 		return NULL;
1199 
1200 	dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1201 	if (dev)
1202 		return to_cxl_port(dev);
1203 	return NULL;
1204 }
1205 
find_cxl_port(struct device * dport_dev,struct cxl_dport ** dport)1206 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1207 				      struct cxl_dport **dport)
1208 {
1209 	struct cxl_find_port_ctx ctx = {
1210 		.dport_dev = dport_dev,
1211 		.dport = dport,
1212 	};
1213 	struct cxl_port *port;
1214 
1215 	port = __find_cxl_port(&ctx);
1216 	return port;
1217 }
1218 
find_cxl_port_at(struct cxl_port * parent_port,struct device * dport_dev,struct cxl_dport ** dport)1219 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1220 					 struct device *dport_dev,
1221 					 struct cxl_dport **dport)
1222 {
1223 	struct cxl_find_port_ctx ctx = {
1224 		.dport_dev = dport_dev,
1225 		.parent_port = parent_port,
1226 		.dport = dport,
1227 	};
1228 	struct cxl_port *port;
1229 
1230 	port = __find_cxl_port(&ctx);
1231 	return port;
1232 }
1233 
1234 /*
1235  * All users of grandparent() are using it to walk PCIe-like switch port
1236  * hierarchy. A PCIe switch is comprised of a bridge device representing the
1237  * upstream switch port and N bridges representing downstream switch ports. When
1238  * bridges stack the grand-parent of a downstream switch port is another
1239  * downstream switch port in the immediate ancestor switch.
1240  */
grandparent(struct device * dev)1241 static struct device *grandparent(struct device *dev)
1242 {
1243 	if (dev && dev->parent)
1244 		return dev->parent->parent;
1245 	return NULL;
1246 }
1247 
endpoint_host(struct cxl_port * endpoint)1248 static struct device *endpoint_host(struct cxl_port *endpoint)
1249 {
1250 	struct cxl_port *port = to_cxl_port(endpoint->dev.parent);
1251 
1252 	if (is_cxl_root(port))
1253 		return port->uport_dev;
1254 	return &port->dev;
1255 }
1256 
delete_endpoint(void * data)1257 static void delete_endpoint(void *data)
1258 {
1259 	struct cxl_memdev *cxlmd = data;
1260 	struct cxl_port *endpoint = cxlmd->endpoint;
1261 	struct device *host = endpoint_host(endpoint);
1262 
1263 	device_lock(host);
1264 	if (host->driver && !endpoint->dead) {
1265 		devm_release_action(host, cxl_unlink_parent_dport, endpoint);
1266 		devm_release_action(host, cxl_unlink_uport, endpoint);
1267 		devm_release_action(host, unregister_port, endpoint);
1268 	}
1269 	cxlmd->endpoint = NULL;
1270 	device_unlock(host);
1271 	put_device(&endpoint->dev);
1272 	put_device(host);
1273 }
1274 
cxl_endpoint_autoremove(struct cxl_memdev * cxlmd,struct cxl_port * endpoint)1275 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1276 {
1277 	struct device *host = endpoint_host(endpoint);
1278 	struct device *dev = &cxlmd->dev;
1279 
1280 	get_device(host);
1281 	get_device(&endpoint->dev);
1282 	cxlmd->endpoint = endpoint;
1283 	cxlmd->depth = endpoint->depth;
1284 	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1285 }
1286 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1287 
1288 /*
1289  * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1290  * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1291  * for a port to be unregistered is when all memdevs beneath that port have gone
1292  * through ->remove(). This "bottom-up" removal selectively removes individual
1293  * child ports manually. This depends on devm_cxl_add_port() to not change is
1294  * devm action registration order, and for dports to have already been
1295  * destroyed by reap_dports().
1296  */
delete_switch_port(struct cxl_port * port)1297 static void delete_switch_port(struct cxl_port *port)
1298 {
1299 	devm_release_action(port->dev.parent, cxl_unlink_parent_dport, port);
1300 	devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1301 	devm_release_action(port->dev.parent, unregister_port, port);
1302 }
1303 
reap_dports(struct cxl_port * port)1304 static void reap_dports(struct cxl_port *port)
1305 {
1306 	struct cxl_dport *dport;
1307 	unsigned long index;
1308 
1309 	device_lock_assert(&port->dev);
1310 
1311 	xa_for_each(&port->dports, index, dport) {
1312 		devm_release_action(&port->dev, cxl_dport_unlink, dport);
1313 		devm_release_action(&port->dev, cxl_dport_remove, dport);
1314 		devm_kfree(&port->dev, dport);
1315 	}
1316 }
1317 
1318 struct detach_ctx {
1319 	struct cxl_memdev *cxlmd;
1320 	int depth;
1321 };
1322 
port_has_memdev(struct device * dev,const void * data)1323 static int port_has_memdev(struct device *dev, const void *data)
1324 {
1325 	const struct detach_ctx *ctx = data;
1326 	struct cxl_port *port;
1327 
1328 	if (!is_cxl_port(dev))
1329 		return 0;
1330 
1331 	port = to_cxl_port(dev);
1332 	if (port->depth != ctx->depth)
1333 		return 0;
1334 
1335 	return !!cxl_ep_load(port, ctx->cxlmd);
1336 }
1337 
cxl_detach_ep(void * data)1338 static void cxl_detach_ep(void *data)
1339 {
1340 	struct cxl_memdev *cxlmd = data;
1341 
1342 	for (int i = cxlmd->depth - 1; i >= 1; i--) {
1343 		struct cxl_port *port, *parent_port;
1344 		struct detach_ctx ctx = {
1345 			.cxlmd = cxlmd,
1346 			.depth = i,
1347 		};
1348 		struct device *dev;
1349 		struct cxl_ep *ep;
1350 		bool died = false;
1351 
1352 		dev = bus_find_device(&cxl_bus_type, NULL, &ctx,
1353 				      port_has_memdev);
1354 		if (!dev)
1355 			continue;
1356 		port = to_cxl_port(dev);
1357 
1358 		parent_port = to_cxl_port(port->dev.parent);
1359 		device_lock(&parent_port->dev);
1360 		device_lock(&port->dev);
1361 		ep = cxl_ep_load(port, cxlmd);
1362 		dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1363 			ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1364 		cxl_ep_remove(port, ep);
1365 		if (ep && !port->dead && xa_empty(&port->endpoints) &&
1366 		    !is_cxl_root(parent_port) && parent_port->dev.driver) {
1367 			/*
1368 			 * This was the last ep attached to a dynamically
1369 			 * enumerated port. Block new cxl_add_ep() and garbage
1370 			 * collect the port.
1371 			 */
1372 			died = true;
1373 			port->dead = true;
1374 			reap_dports(port);
1375 		}
1376 		device_unlock(&port->dev);
1377 
1378 		if (died) {
1379 			dev_dbg(&cxlmd->dev, "delete %s\n",
1380 				dev_name(&port->dev));
1381 			delete_switch_port(port);
1382 		}
1383 		put_device(&port->dev);
1384 		device_unlock(&parent_port->dev);
1385 	}
1386 }
1387 
find_component_registers(struct device * dev)1388 static resource_size_t find_component_registers(struct device *dev)
1389 {
1390 	struct cxl_register_map map;
1391 	struct pci_dev *pdev;
1392 
1393 	/*
1394 	 * Theoretically, CXL component registers can be hosted on a
1395 	 * non-PCI device, in practice, only cxl_test hits this case.
1396 	 */
1397 	if (!dev_is_pci(dev))
1398 		return CXL_RESOURCE_NONE;
1399 
1400 	pdev = to_pci_dev(dev);
1401 
1402 	cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1403 	return map.resource;
1404 }
1405 
add_port_attach_ep(struct cxl_memdev * cxlmd,struct device * uport_dev,struct device * dport_dev)1406 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1407 			      struct device *uport_dev,
1408 			      struct device *dport_dev)
1409 {
1410 	struct device *dparent = grandparent(dport_dev);
1411 	struct cxl_port *port, *parent_port = NULL;
1412 	struct cxl_dport *dport, *parent_dport;
1413 	resource_size_t component_reg_phys;
1414 	int rc;
1415 
1416 	if (!dparent) {
1417 		/*
1418 		 * The iteration reached the topology root without finding the
1419 		 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1420 		 * be re-probed after platform driver attaches.
1421 		 */
1422 		dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1423 			dev_name(dport_dev));
1424 		return -ENXIO;
1425 	}
1426 
1427 	parent_port = find_cxl_port(dparent, &parent_dport);
1428 	if (!parent_port) {
1429 		/* iterate to create this parent_port */
1430 		return -EAGAIN;
1431 	}
1432 
1433 	device_lock(&parent_port->dev);
1434 	if (!parent_port->dev.driver) {
1435 		dev_warn(&cxlmd->dev,
1436 			 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1437 			 dev_name(&parent_port->dev), dev_name(uport_dev));
1438 		port = ERR_PTR(-ENXIO);
1439 		goto out;
1440 	}
1441 
1442 	port = find_cxl_port_at(parent_port, dport_dev, &dport);
1443 	if (!port) {
1444 		component_reg_phys = find_component_registers(uport_dev);
1445 		port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1446 					 component_reg_phys, parent_dport);
1447 		/* retry find to pick up the new dport information */
1448 		if (!IS_ERR(port))
1449 			port = find_cxl_port_at(parent_port, dport_dev, &dport);
1450 	}
1451 out:
1452 	device_unlock(&parent_port->dev);
1453 
1454 	if (IS_ERR(port))
1455 		rc = PTR_ERR(port);
1456 	else {
1457 		dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1458 			dev_name(&port->dev), dev_name(port->uport_dev));
1459 		rc = cxl_add_ep(dport, &cxlmd->dev);
1460 		if (rc == -EBUSY) {
1461 			/*
1462 			 * "can't" happen, but this error code means
1463 			 * something to the caller, so translate it.
1464 			 */
1465 			rc = -ENXIO;
1466 		}
1467 		put_device(&port->dev);
1468 	}
1469 
1470 	put_device(&parent_port->dev);
1471 	return rc;
1472 }
1473 
devm_cxl_enumerate_ports(struct cxl_memdev * cxlmd)1474 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1475 {
1476 	struct device *dev = &cxlmd->dev;
1477 	struct device *iter;
1478 	int rc;
1479 
1480 	/*
1481 	 * Skip intermediate port enumeration in the RCH case, there
1482 	 * are no ports in between a host bridge and an endpoint.
1483 	 */
1484 	if (cxlmd->cxlds->rcd)
1485 		return 0;
1486 
1487 	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1488 	if (rc)
1489 		return rc;
1490 
1491 	/*
1492 	 * Scan for and add all cxl_ports in this device's ancestry.
1493 	 * Repeat until no more ports are added. Abort if a port add
1494 	 * attempt fails.
1495 	 */
1496 retry:
1497 	for (iter = dev; iter; iter = grandparent(iter)) {
1498 		struct device *dport_dev = grandparent(iter);
1499 		struct device *uport_dev;
1500 		struct cxl_dport *dport;
1501 		struct cxl_port *port;
1502 
1503 		if (!dport_dev)
1504 			return 0;
1505 
1506 		uport_dev = dport_dev->parent;
1507 		if (!uport_dev) {
1508 			dev_warn(dev, "at %s no parent for dport: %s\n",
1509 				 dev_name(iter), dev_name(dport_dev));
1510 			return -ENXIO;
1511 		}
1512 
1513 		dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1514 			dev_name(iter), dev_name(dport_dev),
1515 			dev_name(uport_dev));
1516 		port = find_cxl_port(dport_dev, &dport);
1517 		if (port) {
1518 			dev_dbg(&cxlmd->dev,
1519 				"found already registered port %s:%s\n",
1520 				dev_name(&port->dev),
1521 				dev_name(port->uport_dev));
1522 			rc = cxl_add_ep(dport, &cxlmd->dev);
1523 
1524 			/*
1525 			 * If the endpoint already exists in the port's list,
1526 			 * that's ok, it was added on a previous pass.
1527 			 * Otherwise, retry in add_port_attach_ep() after taking
1528 			 * the parent_port lock as the current port may be being
1529 			 * reaped.
1530 			 */
1531 			if (rc && rc != -EBUSY) {
1532 				put_device(&port->dev);
1533 				return rc;
1534 			}
1535 
1536 			/* Any more ports to add between this one and the root? */
1537 			if (!dev_is_cxl_root_child(&port->dev)) {
1538 				put_device(&port->dev);
1539 				continue;
1540 			}
1541 
1542 			put_device(&port->dev);
1543 			return 0;
1544 		}
1545 
1546 		rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1547 		/* port missing, try to add parent */
1548 		if (rc == -EAGAIN)
1549 			continue;
1550 		/* failed to add ep or port */
1551 		if (rc)
1552 			return rc;
1553 		/* port added, new descendants possible, start over */
1554 		goto retry;
1555 	}
1556 
1557 	return 0;
1558 }
1559 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1560 
cxl_pci_find_port(struct pci_dev * pdev,struct cxl_dport ** dport)1561 struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
1562 				   struct cxl_dport **dport)
1563 {
1564 	return find_cxl_port(pdev->dev.parent, dport);
1565 }
1566 EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, CXL);
1567 
cxl_mem_find_port(struct cxl_memdev * cxlmd,struct cxl_dport ** dport)1568 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1569 				   struct cxl_dport **dport)
1570 {
1571 	return find_cxl_port(grandparent(&cxlmd->dev), dport);
1572 }
1573 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1574 
decoder_populate_targets(struct cxl_switch_decoder * cxlsd,struct cxl_port * port,int * target_map)1575 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1576 				    struct cxl_port *port, int *target_map)
1577 {
1578 	int i;
1579 
1580 	if (!target_map)
1581 		return 0;
1582 
1583 	device_lock_assert(&port->dev);
1584 
1585 	if (xa_empty(&port->dports))
1586 		return -EINVAL;
1587 
1588 	guard(rwsem_write)(&cxl_region_rwsem);
1589 	for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
1590 		struct cxl_dport *dport = find_dport(port, target_map[i]);
1591 
1592 		if (!dport)
1593 			return -ENXIO;
1594 		cxlsd->target[i] = dport;
1595 	}
1596 
1597 	return 0;
1598 }
1599 
cxl_hb_modulo(struct cxl_root_decoder * cxlrd,int pos)1600 struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1601 {
1602 	struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1603 	struct cxl_decoder *cxld = &cxlsd->cxld;
1604 	int iw;
1605 
1606 	iw = cxld->interleave_ways;
1607 	if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1608 			  "misconfigured root decoder\n"))
1609 		return NULL;
1610 
1611 	return cxlrd->cxlsd.target[pos % iw];
1612 }
1613 EXPORT_SYMBOL_NS_GPL(cxl_hb_modulo, CXL);
1614 
1615 static struct lock_class_key cxl_decoder_key;
1616 
1617 /**
1618  * cxl_decoder_init - Common decoder setup / initialization
1619  * @port: owning port of this decoder
1620  * @cxld: common decoder properties to initialize
1621  *
1622  * A port may contain one or more decoders. Each of those decoders
1623  * enable some address space for CXL.mem utilization. A decoder is
1624  * expected to be configured by the caller before registering via
1625  * cxl_decoder_add()
1626  */
cxl_decoder_init(struct cxl_port * port,struct cxl_decoder * cxld)1627 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1628 {
1629 	struct device *dev;
1630 	int rc;
1631 
1632 	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1633 	if (rc < 0)
1634 		return rc;
1635 
1636 	/* need parent to stick around to release the id */
1637 	get_device(&port->dev);
1638 	cxld->id = rc;
1639 
1640 	dev = &cxld->dev;
1641 	device_initialize(dev);
1642 	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1643 	device_set_pm_not_required(dev);
1644 	dev->parent = &port->dev;
1645 	dev->bus = &cxl_bus_type;
1646 
1647 	/* Pre initialize an "empty" decoder */
1648 	cxld->interleave_ways = 1;
1649 	cxld->interleave_granularity = PAGE_SIZE;
1650 	cxld->target_type = CXL_DECODER_HOSTONLYMEM;
1651 	cxld->hpa_range = (struct range) {
1652 		.start = 0,
1653 		.end = -1,
1654 	};
1655 
1656 	return 0;
1657 }
1658 
cxl_switch_decoder_init(struct cxl_port * port,struct cxl_switch_decoder * cxlsd,int nr_targets)1659 static int cxl_switch_decoder_init(struct cxl_port *port,
1660 				   struct cxl_switch_decoder *cxlsd,
1661 				   int nr_targets)
1662 {
1663 	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1664 		return -EINVAL;
1665 
1666 	cxlsd->nr_targets = nr_targets;
1667 	return cxl_decoder_init(port, &cxlsd->cxld);
1668 }
1669 
1670 /**
1671  * cxl_root_decoder_alloc - Allocate a root level decoder
1672  * @port: owning CXL root of this decoder
1673  * @nr_targets: static number of downstream targets
1674  * @calc_hb: which host bridge covers the n'th position by granularity
1675  *
1676  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1677  * 'CXL root' decoder is one that decodes from a top-level / static platform
1678  * firmware description of CXL resources into a CXL standard decode
1679  * topology.
1680  */
cxl_root_decoder_alloc(struct cxl_port * port,unsigned int nr_targets,cxl_calc_hb_fn calc_hb)1681 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1682 						unsigned int nr_targets,
1683 						cxl_calc_hb_fn calc_hb)
1684 {
1685 	struct cxl_root_decoder *cxlrd;
1686 	struct cxl_switch_decoder *cxlsd;
1687 	struct cxl_decoder *cxld;
1688 	int rc;
1689 
1690 	if (!is_cxl_root(port))
1691 		return ERR_PTR(-EINVAL);
1692 
1693 	cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1694 			GFP_KERNEL);
1695 	if (!cxlrd)
1696 		return ERR_PTR(-ENOMEM);
1697 
1698 	cxlsd = &cxlrd->cxlsd;
1699 	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1700 	if (rc) {
1701 		kfree(cxlrd);
1702 		return ERR_PTR(rc);
1703 	}
1704 
1705 	cxlrd->calc_hb = calc_hb;
1706 	mutex_init(&cxlrd->range_lock);
1707 
1708 	cxld = &cxlsd->cxld;
1709 	cxld->dev.type = &cxl_decoder_root_type;
1710 	/*
1711 	 * cxl_root_decoder_release() special cases negative ids to
1712 	 * detect memregion_alloc() failures.
1713 	 */
1714 	atomic_set(&cxlrd->region_id, -1);
1715 	rc = memregion_alloc(GFP_KERNEL);
1716 	if (rc < 0) {
1717 		put_device(&cxld->dev);
1718 		return ERR_PTR(rc);
1719 	}
1720 
1721 	atomic_set(&cxlrd->region_id, rc);
1722 	return cxlrd;
1723 }
1724 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1725 
1726 /**
1727  * cxl_switch_decoder_alloc - Allocate a switch level decoder
1728  * @port: owning CXL switch port of this decoder
1729  * @nr_targets: max number of dynamically addressable downstream targets
1730  *
1731  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1732  * 'switch' decoder is any decoder that can be enumerated by PCIe
1733  * topology and the HDM Decoder Capability. This includes the decoders
1734  * that sit between Switch Upstream Ports / Switch Downstream Ports and
1735  * Host Bridges / Root Ports.
1736  */
cxl_switch_decoder_alloc(struct cxl_port * port,unsigned int nr_targets)1737 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1738 						    unsigned int nr_targets)
1739 {
1740 	struct cxl_switch_decoder *cxlsd;
1741 	struct cxl_decoder *cxld;
1742 	int rc;
1743 
1744 	if (is_cxl_root(port) || is_cxl_endpoint(port))
1745 		return ERR_PTR(-EINVAL);
1746 
1747 	cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1748 	if (!cxlsd)
1749 		return ERR_PTR(-ENOMEM);
1750 
1751 	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1752 	if (rc) {
1753 		kfree(cxlsd);
1754 		return ERR_PTR(rc);
1755 	}
1756 
1757 	cxld = &cxlsd->cxld;
1758 	cxld->dev.type = &cxl_decoder_switch_type;
1759 	return cxlsd;
1760 }
1761 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1762 
1763 /**
1764  * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1765  * @port: owning port of this decoder
1766  *
1767  * Return: A new cxl decoder to be registered by cxl_decoder_add()
1768  */
cxl_endpoint_decoder_alloc(struct cxl_port * port)1769 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1770 {
1771 	struct cxl_endpoint_decoder *cxled;
1772 	struct cxl_decoder *cxld;
1773 	int rc;
1774 
1775 	if (!is_cxl_endpoint(port))
1776 		return ERR_PTR(-EINVAL);
1777 
1778 	cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1779 	if (!cxled)
1780 		return ERR_PTR(-ENOMEM);
1781 
1782 	cxled->pos = -1;
1783 	cxld = &cxled->cxld;
1784 	rc = cxl_decoder_init(port, cxld);
1785 	if (rc)	 {
1786 		kfree(cxled);
1787 		return ERR_PTR(rc);
1788 	}
1789 
1790 	cxld->dev.type = &cxl_decoder_endpoint_type;
1791 	return cxled;
1792 }
1793 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1794 
1795 /**
1796  * cxl_decoder_add_locked - Add a decoder with targets
1797  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1798  * @target_map: A list of downstream ports that this decoder can direct memory
1799  *              traffic to. These numbers should correspond with the port number
1800  *              in the PCIe Link Capabilities structure.
1801  *
1802  * Certain types of decoders may not have any targets. The main example of this
1803  * is an endpoint device. A more awkward example is a hostbridge whose root
1804  * ports get hot added (technically possible, though unlikely).
1805  *
1806  * This is the locked variant of cxl_decoder_add().
1807  *
1808  * Context: Process context. Expects the device lock of the port that owns the
1809  *	    @cxld to be held.
1810  *
1811  * Return: Negative error code if the decoder wasn't properly configured; else
1812  *	   returns 0.
1813  */
cxl_decoder_add_locked(struct cxl_decoder * cxld,int * target_map)1814 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1815 {
1816 	struct cxl_port *port;
1817 	struct device *dev;
1818 	int rc;
1819 
1820 	if (WARN_ON_ONCE(!cxld))
1821 		return -EINVAL;
1822 
1823 	if (WARN_ON_ONCE(IS_ERR(cxld)))
1824 		return PTR_ERR(cxld);
1825 
1826 	if (cxld->interleave_ways < 1)
1827 		return -EINVAL;
1828 
1829 	dev = &cxld->dev;
1830 
1831 	port = to_cxl_port(cxld->dev.parent);
1832 	if (!is_endpoint_decoder(dev)) {
1833 		struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1834 
1835 		rc = decoder_populate_targets(cxlsd, port, target_map);
1836 		if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1837 			dev_err(&port->dev,
1838 				"Failed to populate active decoder targets\n");
1839 			return rc;
1840 		}
1841 	}
1842 
1843 	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1844 	if (rc)
1845 		return rc;
1846 
1847 	return device_add(dev);
1848 }
1849 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1850 
1851 /**
1852  * cxl_decoder_add - Add a decoder with targets
1853  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1854  * @target_map: A list of downstream ports that this decoder can direct memory
1855  *              traffic to. These numbers should correspond with the port number
1856  *              in the PCIe Link Capabilities structure.
1857  *
1858  * This is the unlocked variant of cxl_decoder_add_locked().
1859  * See cxl_decoder_add_locked().
1860  *
1861  * Context: Process context. Takes and releases the device lock of the port that
1862  *	    owns the @cxld.
1863  */
cxl_decoder_add(struct cxl_decoder * cxld,int * target_map)1864 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1865 {
1866 	struct cxl_port *port;
1867 	int rc;
1868 
1869 	if (WARN_ON_ONCE(!cxld))
1870 		return -EINVAL;
1871 
1872 	if (WARN_ON_ONCE(IS_ERR(cxld)))
1873 		return PTR_ERR(cxld);
1874 
1875 	port = to_cxl_port(cxld->dev.parent);
1876 
1877 	device_lock(&port->dev);
1878 	rc = cxl_decoder_add_locked(cxld, target_map);
1879 	device_unlock(&port->dev);
1880 
1881 	return rc;
1882 }
1883 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1884 
cxld_unregister(void * dev)1885 static void cxld_unregister(void *dev)
1886 {
1887 	struct cxl_endpoint_decoder *cxled;
1888 
1889 	if (is_endpoint_decoder(dev)) {
1890 		cxled = to_cxl_endpoint_decoder(dev);
1891 		cxl_decoder_kill_region(cxled);
1892 	}
1893 
1894 	device_unregister(dev);
1895 }
1896 
cxl_decoder_autoremove(struct device * host,struct cxl_decoder * cxld)1897 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1898 {
1899 	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1900 }
1901 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1902 
1903 /**
1904  * __cxl_driver_register - register a driver for the cxl bus
1905  * @cxl_drv: cxl driver structure to attach
1906  * @owner: owning module/driver
1907  * @modname: KBUILD_MODNAME for parent driver
1908  */
__cxl_driver_register(struct cxl_driver * cxl_drv,struct module * owner,const char * modname)1909 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1910 			  const char *modname)
1911 {
1912 	if (!cxl_drv->probe) {
1913 		pr_debug("%s ->probe() must be specified\n", modname);
1914 		return -EINVAL;
1915 	}
1916 
1917 	if (!cxl_drv->name) {
1918 		pr_debug("%s ->name must be specified\n", modname);
1919 		return -EINVAL;
1920 	}
1921 
1922 	if (!cxl_drv->id) {
1923 		pr_debug("%s ->id must be specified\n", modname);
1924 		return -EINVAL;
1925 	}
1926 
1927 	cxl_drv->drv.bus = &cxl_bus_type;
1928 	cxl_drv->drv.owner = owner;
1929 	cxl_drv->drv.mod_name = modname;
1930 	cxl_drv->drv.name = cxl_drv->name;
1931 
1932 	return driver_register(&cxl_drv->drv);
1933 }
1934 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1935 
cxl_driver_unregister(struct cxl_driver * cxl_drv)1936 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1937 {
1938 	driver_unregister(&cxl_drv->drv);
1939 }
1940 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1941 
cxl_bus_uevent(const struct device * dev,struct kobj_uevent_env * env)1942 static int cxl_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
1943 {
1944 	return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1945 			      cxl_device_id(dev));
1946 }
1947 
cxl_bus_match(struct device * dev,struct device_driver * drv)1948 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1949 {
1950 	return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1951 }
1952 
cxl_bus_probe(struct device * dev)1953 static int cxl_bus_probe(struct device *dev)
1954 {
1955 	int rc;
1956 
1957 	rc = to_cxl_drv(dev->driver)->probe(dev);
1958 	dev_dbg(dev, "probe: %d\n", rc);
1959 	return rc;
1960 }
1961 
cxl_bus_remove(struct device * dev)1962 static void cxl_bus_remove(struct device *dev)
1963 {
1964 	struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1965 
1966 	if (cxl_drv->remove)
1967 		cxl_drv->remove(dev);
1968 }
1969 
1970 static struct workqueue_struct *cxl_bus_wq;
1971 
cxl_bus_rescan_queue(struct work_struct * w)1972 static void cxl_bus_rescan_queue(struct work_struct *w)
1973 {
1974 	int rc = bus_rescan_devices(&cxl_bus_type);
1975 
1976 	pr_debug("CXL bus rescan result: %d\n", rc);
1977 }
1978 
cxl_bus_rescan(void)1979 void cxl_bus_rescan(void)
1980 {
1981 	static DECLARE_WORK(rescan_work, cxl_bus_rescan_queue);
1982 
1983 	queue_work(cxl_bus_wq, &rescan_work);
1984 }
1985 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1986 
cxl_bus_drain(void)1987 void cxl_bus_drain(void)
1988 {
1989 	drain_workqueue(cxl_bus_wq);
1990 }
1991 EXPORT_SYMBOL_NS_GPL(cxl_bus_drain, CXL);
1992 
schedule_cxl_memdev_detach(struct cxl_memdev * cxlmd)1993 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1994 {
1995 	return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1996 }
1997 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1998 
1999 /* for user tooling to ensure port disable work has completed */
flush_store(const struct bus_type * bus,const char * buf,size_t count)2000 static ssize_t flush_store(const struct bus_type *bus, const char *buf, size_t count)
2001 {
2002 	if (sysfs_streq(buf, "1")) {
2003 		flush_workqueue(cxl_bus_wq);
2004 		return count;
2005 	}
2006 
2007 	return -EINVAL;
2008 }
2009 
2010 static BUS_ATTR_WO(flush);
2011 
2012 static struct attribute *cxl_bus_attributes[] = {
2013 	&bus_attr_flush.attr,
2014 	NULL,
2015 };
2016 
2017 static struct attribute_group cxl_bus_attribute_group = {
2018 	.attrs = cxl_bus_attributes,
2019 };
2020 
2021 static const struct attribute_group *cxl_bus_attribute_groups[] = {
2022 	&cxl_bus_attribute_group,
2023 	NULL,
2024 };
2025 
2026 struct bus_type cxl_bus_type = {
2027 	.name = "cxl",
2028 	.uevent = cxl_bus_uevent,
2029 	.match = cxl_bus_match,
2030 	.probe = cxl_bus_probe,
2031 	.remove = cxl_bus_remove,
2032 	.bus_groups = cxl_bus_attribute_groups,
2033 };
2034 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
2035 
2036 static struct dentry *cxl_debugfs;
2037 
cxl_debugfs_create_dir(const char * dir)2038 struct dentry *cxl_debugfs_create_dir(const char *dir)
2039 {
2040 	return debugfs_create_dir(dir, cxl_debugfs);
2041 }
2042 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
2043 
cxl_core_init(void)2044 static __init int cxl_core_init(void)
2045 {
2046 	int rc;
2047 
2048 	cxl_debugfs = debugfs_create_dir("cxl", NULL);
2049 
2050 	cxl_mbox_init();
2051 
2052 	rc = cxl_memdev_init();
2053 	if (rc)
2054 		return rc;
2055 
2056 	cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
2057 	if (!cxl_bus_wq) {
2058 		rc = -ENOMEM;
2059 		goto err_wq;
2060 	}
2061 
2062 	rc = bus_register(&cxl_bus_type);
2063 	if (rc)
2064 		goto err_bus;
2065 
2066 	rc = cxl_region_init();
2067 	if (rc)
2068 		goto err_region;
2069 
2070 	return 0;
2071 
2072 err_region:
2073 	bus_unregister(&cxl_bus_type);
2074 err_bus:
2075 	destroy_workqueue(cxl_bus_wq);
2076 err_wq:
2077 	cxl_memdev_exit();
2078 	return rc;
2079 }
2080 
cxl_core_exit(void)2081 static void cxl_core_exit(void)
2082 {
2083 	cxl_region_exit();
2084 	bus_unregister(&cxl_bus_type);
2085 	destroy_workqueue(cxl_bus_wq);
2086 	cxl_memdev_exit();
2087 	debugfs_remove_recursive(cxl_debugfs);
2088 }
2089 
2090 subsys_initcall(cxl_core_init);
2091 module_exit(cxl_core_exit);
2092 MODULE_LICENSE("GPL v2");
2093