1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
3 #include <linux/io-64-nonatomic-lo-hi.h>
4 #include <linux/memregion.h>
5 #include <linux/workqueue.h>
6 #include <linux/debugfs.h>
7 #include <linux/device.h>
8 #include <linux/module.h>
9 #include <linux/pci.h>
10 #include <linux/slab.h>
11 #include <linux/idr.h>
12 #include <cxlmem.h>
13 #include <cxlpci.h>
14 #include <cxl.h>
15 #include "core.h"
16 
17 /**
18  * DOC: cxl core
19  *
20  * The CXL core provides a set of interfaces that can be consumed by CXL aware
21  * drivers. The interfaces allow for creation, modification, and destruction of
22  * regions, memory devices, ports, and decoders. CXL aware drivers must register
23  * with the CXL core via these interfaces in order to be able to participate in
24  * cross-device interleave coordination. The CXL core also establishes and
25  * maintains the bridge to the nvdimm subsystem.
26  *
27  * CXL core introduces sysfs hierarchy to control the devices that are
28  * instantiated by the core.
29  */
30 
31 static DEFINE_IDA(cxl_port_ida);
32 static DEFINE_XARRAY(cxl_root_buses);
33 
devtype_show(struct device * dev,struct device_attribute * attr,char * buf)34 static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
35 			    char *buf)
36 {
37 	return sysfs_emit(buf, "%s\n", dev->type->name);
38 }
39 static DEVICE_ATTR_RO(devtype);
40 
cxl_device_id(struct device * dev)41 static int cxl_device_id(struct device *dev)
42 {
43 	if (dev->type == &cxl_nvdimm_bridge_type)
44 		return CXL_DEVICE_NVDIMM_BRIDGE;
45 	if (dev->type == &cxl_nvdimm_type)
46 		return CXL_DEVICE_NVDIMM;
47 	if (dev->type == CXL_PMEM_REGION_TYPE())
48 		return CXL_DEVICE_PMEM_REGION;
49 	if (is_cxl_port(dev)) {
50 		if (is_cxl_root(to_cxl_port(dev)))
51 			return CXL_DEVICE_ROOT;
52 		return CXL_DEVICE_PORT;
53 	}
54 	if (is_cxl_memdev(dev))
55 		return CXL_DEVICE_MEMORY_EXPANDER;
56 	if (dev->type == CXL_REGION_TYPE())
57 		return CXL_DEVICE_REGION;
58 	return 0;
59 }
60 
modalias_show(struct device * dev,struct device_attribute * attr,char * buf)61 static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
62 			     char *buf)
63 {
64 	return sysfs_emit(buf, CXL_MODALIAS_FMT "\n", cxl_device_id(dev));
65 }
66 static DEVICE_ATTR_RO(modalias);
67 
68 static struct attribute *cxl_base_attributes[] = {
69 	&dev_attr_devtype.attr,
70 	&dev_attr_modalias.attr,
71 	NULL,
72 };
73 
74 struct attribute_group cxl_base_attribute_group = {
75 	.attrs = cxl_base_attributes,
76 };
77 
start_show(struct device * dev,struct device_attribute * attr,char * buf)78 static ssize_t start_show(struct device *dev, struct device_attribute *attr,
79 			  char *buf)
80 {
81 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
82 
83 	return sysfs_emit(buf, "%#llx\n", cxld->hpa_range.start);
84 }
85 static DEVICE_ATTR_ADMIN_RO(start);
86 
size_show(struct device * dev,struct device_attribute * attr,char * buf)87 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
88 			char *buf)
89 {
90 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
91 
92 	return sysfs_emit(buf, "%#llx\n", range_len(&cxld->hpa_range));
93 }
94 static DEVICE_ATTR_RO(size);
95 
96 #define CXL_DECODER_FLAG_ATTR(name, flag)                            \
97 static ssize_t name##_show(struct device *dev,                       \
98 			   struct device_attribute *attr, char *buf) \
99 {                                                                    \
100 	struct cxl_decoder *cxld = to_cxl_decoder(dev);              \
101                                                                      \
102 	return sysfs_emit(buf, "%s\n",                               \
103 			  (cxld->flags & (flag)) ? "1" : "0");       \
104 }                                                                    \
105 static DEVICE_ATTR_RO(name)
106 
107 CXL_DECODER_FLAG_ATTR(cap_pmem, CXL_DECODER_F_PMEM);
108 CXL_DECODER_FLAG_ATTR(cap_ram, CXL_DECODER_F_RAM);
109 CXL_DECODER_FLAG_ATTR(cap_type2, CXL_DECODER_F_TYPE2);
110 CXL_DECODER_FLAG_ATTR(cap_type3, CXL_DECODER_F_TYPE3);
111 CXL_DECODER_FLAG_ATTR(locked, CXL_DECODER_F_LOCK);
112 
target_type_show(struct device * dev,struct device_attribute * attr,char * buf)113 static ssize_t target_type_show(struct device *dev,
114 				struct device_attribute *attr, char *buf)
115 {
116 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
117 
118 	switch (cxld->target_type) {
119 	case CXL_DECODER_ACCELERATOR:
120 		return sysfs_emit(buf, "accelerator\n");
121 	case CXL_DECODER_EXPANDER:
122 		return sysfs_emit(buf, "expander\n");
123 	}
124 	return -ENXIO;
125 }
126 static DEVICE_ATTR_RO(target_type);
127 
emit_target_list(struct cxl_switch_decoder * cxlsd,char * buf)128 static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
129 {
130 	struct cxl_decoder *cxld = &cxlsd->cxld;
131 	ssize_t offset = 0;
132 	int i, rc = 0;
133 
134 	for (i = 0; i < cxld->interleave_ways; i++) {
135 		struct cxl_dport *dport = cxlsd->target[i];
136 		struct cxl_dport *next = NULL;
137 
138 		if (!dport)
139 			break;
140 
141 		if (i + 1 < cxld->interleave_ways)
142 			next = cxlsd->target[i + 1];
143 		rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
144 				   next ? "," : "");
145 		if (rc < 0)
146 			return rc;
147 		offset += rc;
148 	}
149 
150 	return offset;
151 }
152 
target_list_show(struct device * dev,struct device_attribute * attr,char * buf)153 static ssize_t target_list_show(struct device *dev,
154 				struct device_attribute *attr, char *buf)
155 {
156 	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
157 	ssize_t offset;
158 	unsigned int seq;
159 	int rc;
160 
161 	do {
162 		seq = read_seqbegin(&cxlsd->target_lock);
163 		rc = emit_target_list(cxlsd, buf);
164 	} while (read_seqretry(&cxlsd->target_lock, seq));
165 
166 	if (rc < 0)
167 		return rc;
168 	offset = rc;
169 
170 	rc = sysfs_emit_at(buf, offset, "\n");
171 	if (rc < 0)
172 		return rc;
173 
174 	return offset + rc;
175 }
176 static DEVICE_ATTR_RO(target_list);
177 
mode_show(struct device * dev,struct device_attribute * attr,char * buf)178 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
179 			 char *buf)
180 {
181 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
182 
183 	switch (cxled->mode) {
184 	case CXL_DECODER_RAM:
185 		return sysfs_emit(buf, "ram\n");
186 	case CXL_DECODER_PMEM:
187 		return sysfs_emit(buf, "pmem\n");
188 	case CXL_DECODER_NONE:
189 		return sysfs_emit(buf, "none\n");
190 	case CXL_DECODER_MIXED:
191 	default:
192 		return sysfs_emit(buf, "mixed\n");
193 	}
194 }
195 
mode_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)196 static ssize_t mode_store(struct device *dev, struct device_attribute *attr,
197 			  const char *buf, size_t len)
198 {
199 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
200 	enum cxl_decoder_mode mode;
201 	ssize_t rc;
202 
203 	if (sysfs_streq(buf, "pmem"))
204 		mode = CXL_DECODER_PMEM;
205 	else if (sysfs_streq(buf, "ram"))
206 		mode = CXL_DECODER_RAM;
207 	else
208 		return -EINVAL;
209 
210 	rc = cxl_dpa_set_mode(cxled, mode);
211 	if (rc)
212 		return rc;
213 
214 	return len;
215 }
216 static DEVICE_ATTR_RW(mode);
217 
dpa_resource_show(struct device * dev,struct device_attribute * attr,char * buf)218 static ssize_t dpa_resource_show(struct device *dev, struct device_attribute *attr,
219 			    char *buf)
220 {
221 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
222 	u64 base = cxl_dpa_resource_start(cxled);
223 
224 	return sysfs_emit(buf, "%#llx\n", base);
225 }
226 static DEVICE_ATTR_RO(dpa_resource);
227 
dpa_size_show(struct device * dev,struct device_attribute * attr,char * buf)228 static ssize_t dpa_size_show(struct device *dev, struct device_attribute *attr,
229 			     char *buf)
230 {
231 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
232 	resource_size_t size = cxl_dpa_size(cxled);
233 
234 	return sysfs_emit(buf, "%pa\n", &size);
235 }
236 
dpa_size_store(struct device * dev,struct device_attribute * attr,const char * buf,size_t len)237 static ssize_t dpa_size_store(struct device *dev, struct device_attribute *attr,
238 			      const char *buf, size_t len)
239 {
240 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
241 	unsigned long long size;
242 	ssize_t rc;
243 
244 	rc = kstrtoull(buf, 0, &size);
245 	if (rc)
246 		return rc;
247 
248 	if (!IS_ALIGNED(size, SZ_256M))
249 		return -EINVAL;
250 
251 	rc = cxl_dpa_free(cxled);
252 	if (rc)
253 		return rc;
254 
255 	if (size == 0)
256 		return len;
257 
258 	rc = cxl_dpa_alloc(cxled, size);
259 	if (rc)
260 		return rc;
261 
262 	return len;
263 }
264 static DEVICE_ATTR_RW(dpa_size);
265 
interleave_granularity_show(struct device * dev,struct device_attribute * attr,char * buf)266 static ssize_t interleave_granularity_show(struct device *dev,
267 					   struct device_attribute *attr,
268 					   char *buf)
269 {
270 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
271 
272 	return sysfs_emit(buf, "%d\n", cxld->interleave_granularity);
273 }
274 
275 static DEVICE_ATTR_RO(interleave_granularity);
276 
interleave_ways_show(struct device * dev,struct device_attribute * attr,char * buf)277 static ssize_t interleave_ways_show(struct device *dev,
278 				    struct device_attribute *attr, char *buf)
279 {
280 	struct cxl_decoder *cxld = to_cxl_decoder(dev);
281 
282 	return sysfs_emit(buf, "%d\n", cxld->interleave_ways);
283 }
284 
285 static DEVICE_ATTR_RO(interleave_ways);
286 
287 static struct attribute *cxl_decoder_base_attrs[] = {
288 	&dev_attr_start.attr,
289 	&dev_attr_size.attr,
290 	&dev_attr_locked.attr,
291 	&dev_attr_interleave_granularity.attr,
292 	&dev_attr_interleave_ways.attr,
293 	NULL,
294 };
295 
296 static struct attribute_group cxl_decoder_base_attribute_group = {
297 	.attrs = cxl_decoder_base_attrs,
298 };
299 
300 static struct attribute *cxl_decoder_root_attrs[] = {
301 	&dev_attr_cap_pmem.attr,
302 	&dev_attr_cap_ram.attr,
303 	&dev_attr_cap_type2.attr,
304 	&dev_attr_cap_type3.attr,
305 	&dev_attr_target_list.attr,
306 	SET_CXL_REGION_ATTR(create_pmem_region)
307 	SET_CXL_REGION_ATTR(delete_region)
308 	NULL,
309 };
310 
can_create_pmem(struct cxl_root_decoder * cxlrd)311 static bool can_create_pmem(struct cxl_root_decoder *cxlrd)
312 {
313 	unsigned long flags = CXL_DECODER_F_TYPE3 | CXL_DECODER_F_PMEM;
314 
315 	return (cxlrd->cxlsd.cxld.flags & flags) == flags;
316 }
317 
cxl_root_decoder_visible(struct kobject * kobj,struct attribute * a,int n)318 static umode_t cxl_root_decoder_visible(struct kobject *kobj, struct attribute *a, int n)
319 {
320 	struct device *dev = kobj_to_dev(kobj);
321 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
322 
323 	if (a == CXL_REGION_ATTR(create_pmem_region) && !can_create_pmem(cxlrd))
324 		return 0;
325 
326 	if (a == CXL_REGION_ATTR(delete_region) && !can_create_pmem(cxlrd))
327 		return 0;
328 
329 	return a->mode;
330 }
331 
332 static struct attribute_group cxl_decoder_root_attribute_group = {
333 	.attrs = cxl_decoder_root_attrs,
334 	.is_visible = cxl_root_decoder_visible,
335 };
336 
337 static const struct attribute_group *cxl_decoder_root_attribute_groups[] = {
338 	&cxl_decoder_root_attribute_group,
339 	&cxl_decoder_base_attribute_group,
340 	&cxl_base_attribute_group,
341 	NULL,
342 };
343 
344 static struct attribute *cxl_decoder_switch_attrs[] = {
345 	&dev_attr_target_type.attr,
346 	&dev_attr_target_list.attr,
347 	SET_CXL_REGION_ATTR(region)
348 	NULL,
349 };
350 
351 static struct attribute_group cxl_decoder_switch_attribute_group = {
352 	.attrs = cxl_decoder_switch_attrs,
353 };
354 
355 static const struct attribute_group *cxl_decoder_switch_attribute_groups[] = {
356 	&cxl_decoder_switch_attribute_group,
357 	&cxl_decoder_base_attribute_group,
358 	&cxl_base_attribute_group,
359 	NULL,
360 };
361 
362 static struct attribute *cxl_decoder_endpoint_attrs[] = {
363 	&dev_attr_target_type.attr,
364 	&dev_attr_mode.attr,
365 	&dev_attr_dpa_size.attr,
366 	&dev_attr_dpa_resource.attr,
367 	SET_CXL_REGION_ATTR(region)
368 	NULL,
369 };
370 
371 static struct attribute_group cxl_decoder_endpoint_attribute_group = {
372 	.attrs = cxl_decoder_endpoint_attrs,
373 };
374 
375 static const struct attribute_group *cxl_decoder_endpoint_attribute_groups[] = {
376 	&cxl_decoder_base_attribute_group,
377 	&cxl_decoder_endpoint_attribute_group,
378 	&cxl_base_attribute_group,
379 	NULL,
380 };
381 
__cxl_decoder_release(struct cxl_decoder * cxld)382 static void __cxl_decoder_release(struct cxl_decoder *cxld)
383 {
384 	struct cxl_port *port = to_cxl_port(cxld->dev.parent);
385 
386 	ida_free(&port->decoder_ida, cxld->id);
387 	put_device(&port->dev);
388 }
389 
cxl_endpoint_decoder_release(struct device * dev)390 static void cxl_endpoint_decoder_release(struct device *dev)
391 {
392 	struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(dev);
393 
394 	__cxl_decoder_release(&cxled->cxld);
395 	kfree(cxled);
396 }
397 
cxl_switch_decoder_release(struct device * dev)398 static void cxl_switch_decoder_release(struct device *dev)
399 {
400 	struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
401 
402 	__cxl_decoder_release(&cxlsd->cxld);
403 	kfree(cxlsd);
404 }
405 
to_cxl_root_decoder(struct device * dev)406 struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev)
407 {
408 	if (dev_WARN_ONCE(dev, !is_root_decoder(dev),
409 			  "not a cxl_root_decoder device\n"))
410 		return NULL;
411 	return container_of(dev, struct cxl_root_decoder, cxlsd.cxld.dev);
412 }
413 EXPORT_SYMBOL_NS_GPL(to_cxl_root_decoder, CXL);
414 
cxl_root_decoder_release(struct device * dev)415 static void cxl_root_decoder_release(struct device *dev)
416 {
417 	struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
418 
419 	if (atomic_read(&cxlrd->region_id) >= 0)
420 		memregion_free(atomic_read(&cxlrd->region_id));
421 	__cxl_decoder_release(&cxlrd->cxlsd.cxld);
422 	kfree(cxlrd);
423 }
424 
425 static const struct device_type cxl_decoder_endpoint_type = {
426 	.name = "cxl_decoder_endpoint",
427 	.release = cxl_endpoint_decoder_release,
428 	.groups = cxl_decoder_endpoint_attribute_groups,
429 };
430 
431 static const struct device_type cxl_decoder_switch_type = {
432 	.name = "cxl_decoder_switch",
433 	.release = cxl_switch_decoder_release,
434 	.groups = cxl_decoder_switch_attribute_groups,
435 };
436 
437 static const struct device_type cxl_decoder_root_type = {
438 	.name = "cxl_decoder_root",
439 	.release = cxl_root_decoder_release,
440 	.groups = cxl_decoder_root_attribute_groups,
441 };
442 
is_endpoint_decoder(struct device * dev)443 bool is_endpoint_decoder(struct device *dev)
444 {
445 	return dev->type == &cxl_decoder_endpoint_type;
446 }
447 
is_root_decoder(struct device * dev)448 bool is_root_decoder(struct device *dev)
449 {
450 	return dev->type == &cxl_decoder_root_type;
451 }
452 EXPORT_SYMBOL_NS_GPL(is_root_decoder, CXL);
453 
is_switch_decoder(struct device * dev)454 bool is_switch_decoder(struct device *dev)
455 {
456 	return is_root_decoder(dev) || dev->type == &cxl_decoder_switch_type;
457 }
458 
to_cxl_decoder(struct device * dev)459 struct cxl_decoder *to_cxl_decoder(struct device *dev)
460 {
461 	if (dev_WARN_ONCE(dev,
462 			  !is_switch_decoder(dev) && !is_endpoint_decoder(dev),
463 			  "not a cxl_decoder device\n"))
464 		return NULL;
465 	return container_of(dev, struct cxl_decoder, dev);
466 }
467 EXPORT_SYMBOL_NS_GPL(to_cxl_decoder, CXL);
468 
to_cxl_endpoint_decoder(struct device * dev)469 struct cxl_endpoint_decoder *to_cxl_endpoint_decoder(struct device *dev)
470 {
471 	if (dev_WARN_ONCE(dev, !is_endpoint_decoder(dev),
472 			  "not a cxl_endpoint_decoder device\n"))
473 		return NULL;
474 	return container_of(dev, struct cxl_endpoint_decoder, cxld.dev);
475 }
476 EXPORT_SYMBOL_NS_GPL(to_cxl_endpoint_decoder, CXL);
477 
to_cxl_switch_decoder(struct device * dev)478 struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev)
479 {
480 	if (dev_WARN_ONCE(dev, !is_switch_decoder(dev),
481 			  "not a cxl_switch_decoder device\n"))
482 		return NULL;
483 	return container_of(dev, struct cxl_switch_decoder, cxld.dev);
484 }
485 
cxl_ep_release(struct cxl_ep * ep)486 static void cxl_ep_release(struct cxl_ep *ep)
487 {
488 	put_device(ep->ep);
489 	kfree(ep);
490 }
491 
cxl_ep_remove(struct cxl_port * port,struct cxl_ep * ep)492 static void cxl_ep_remove(struct cxl_port *port, struct cxl_ep *ep)
493 {
494 	if (!ep)
495 		return;
496 	xa_erase(&port->endpoints, (unsigned long) ep->ep);
497 	cxl_ep_release(ep);
498 }
499 
cxl_port_release(struct device * dev)500 static void cxl_port_release(struct device *dev)
501 {
502 	struct cxl_port *port = to_cxl_port(dev);
503 	unsigned long index;
504 	struct cxl_ep *ep;
505 
506 	xa_for_each(&port->endpoints, index, ep)
507 		cxl_ep_remove(port, ep);
508 	xa_destroy(&port->endpoints);
509 	xa_destroy(&port->dports);
510 	xa_destroy(&port->regions);
511 	ida_free(&cxl_port_ida, port->id);
512 	kfree(port);
513 }
514 
515 static const struct attribute_group *cxl_port_attribute_groups[] = {
516 	&cxl_base_attribute_group,
517 	NULL,
518 };
519 
520 static const struct device_type cxl_port_type = {
521 	.name = "cxl_port",
522 	.release = cxl_port_release,
523 	.groups = cxl_port_attribute_groups,
524 };
525 
is_cxl_port(struct device * dev)526 bool is_cxl_port(struct device *dev)
527 {
528 	return dev->type == &cxl_port_type;
529 }
530 EXPORT_SYMBOL_NS_GPL(is_cxl_port, CXL);
531 
to_cxl_port(struct device * dev)532 struct cxl_port *to_cxl_port(struct device *dev)
533 {
534 	if (dev_WARN_ONCE(dev, dev->type != &cxl_port_type,
535 			  "not a cxl_port device\n"))
536 		return NULL;
537 	return container_of(dev, struct cxl_port, dev);
538 }
539 EXPORT_SYMBOL_NS_GPL(to_cxl_port, CXL);
540 
unregister_port(void * _port)541 static void unregister_port(void *_port)
542 {
543 	struct cxl_port *port = _port;
544 	struct cxl_port *parent;
545 	struct device *lock_dev;
546 
547 	if (is_cxl_root(port))
548 		parent = NULL;
549 	else
550 		parent = to_cxl_port(port->dev.parent);
551 
552 	/*
553 	 * CXL root port's and the first level of ports are unregistered
554 	 * under the platform firmware device lock, all other ports are
555 	 * unregistered while holding their parent port lock.
556 	 */
557 	if (!parent)
558 		lock_dev = port->uport;
559 	else if (is_cxl_root(parent))
560 		lock_dev = parent->uport;
561 	else
562 		lock_dev = &parent->dev;
563 
564 	device_lock_assert(lock_dev);
565 	port->dead = true;
566 	device_unregister(&port->dev);
567 }
568 
cxl_unlink_uport(void * _port)569 static void cxl_unlink_uport(void *_port)
570 {
571 	struct cxl_port *port = _port;
572 
573 	sysfs_remove_link(&port->dev.kobj, "uport");
574 }
575 
devm_cxl_link_uport(struct device * host,struct cxl_port * port)576 static int devm_cxl_link_uport(struct device *host, struct cxl_port *port)
577 {
578 	int rc;
579 
580 	rc = sysfs_create_link(&port->dev.kobj, &port->uport->kobj, "uport");
581 	if (rc)
582 		return rc;
583 	return devm_add_action_or_reset(host, cxl_unlink_uport, port);
584 }
585 
586 static struct lock_class_key cxl_port_key;
587 
cxl_port_alloc(struct device * uport,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)588 static struct cxl_port *cxl_port_alloc(struct device *uport,
589 				       resource_size_t component_reg_phys,
590 				       struct cxl_dport *parent_dport)
591 {
592 	struct cxl_port *port;
593 	struct device *dev;
594 	int rc;
595 
596 	port = kzalloc(sizeof(*port), GFP_KERNEL);
597 	if (!port)
598 		return ERR_PTR(-ENOMEM);
599 
600 	rc = ida_alloc(&cxl_port_ida, GFP_KERNEL);
601 	if (rc < 0)
602 		goto err;
603 	port->id = rc;
604 	port->uport = uport;
605 
606 	/*
607 	 * The top-level cxl_port "cxl_root" does not have a cxl_port as
608 	 * its parent and it does not have any corresponding component
609 	 * registers as its decode is described by a fixed platform
610 	 * description.
611 	 */
612 	dev = &port->dev;
613 	if (parent_dport) {
614 		struct cxl_port *parent_port = parent_dport->port;
615 		struct cxl_port *iter;
616 
617 		dev->parent = &parent_port->dev;
618 		port->depth = parent_port->depth + 1;
619 		port->parent_dport = parent_dport;
620 
621 		/*
622 		 * walk to the host bridge, or the first ancestor that knows
623 		 * the host bridge
624 		 */
625 		iter = port;
626 		while (!iter->host_bridge &&
627 		       !is_cxl_root(to_cxl_port(iter->dev.parent)))
628 			iter = to_cxl_port(iter->dev.parent);
629 		if (iter->host_bridge)
630 			port->host_bridge = iter->host_bridge;
631 		else
632 			port->host_bridge = iter->uport;
633 		dev_dbg(uport, "host-bridge: %s\n", dev_name(port->host_bridge));
634 	} else
635 		dev->parent = uport;
636 
637 	port->component_reg_phys = component_reg_phys;
638 	ida_init(&port->decoder_ida);
639 	port->hdm_end = -1;
640 	port->commit_end = -1;
641 	xa_init(&port->dports);
642 	xa_init(&port->endpoints);
643 	xa_init(&port->regions);
644 
645 	device_initialize(dev);
646 	lockdep_set_class_and_subclass(&dev->mutex, &cxl_port_key, port->depth);
647 	device_set_pm_not_required(dev);
648 	dev->bus = &cxl_bus_type;
649 	dev->type = &cxl_port_type;
650 
651 	return port;
652 
653 err:
654 	kfree(port);
655 	return ERR_PTR(rc);
656 }
657 
658 /**
659  * devm_cxl_add_port - register a cxl_port in CXL memory decode hierarchy
660  * @host: host device for devm operations
661  * @uport: "physical" device implementing this upstream port
662  * @component_reg_phys: (optional) for configurable cxl_port instances
663  * @parent_dport: next hop up in the CXL memory decode hierarchy
664  */
devm_cxl_add_port(struct device * host,struct device * uport,resource_size_t component_reg_phys,struct cxl_dport * parent_dport)665 struct cxl_port *devm_cxl_add_port(struct device *host, struct device *uport,
666 				   resource_size_t component_reg_phys,
667 				   struct cxl_dport *parent_dport)
668 {
669 	struct cxl_port *port;
670 	struct device *dev;
671 	int rc;
672 
673 	port = cxl_port_alloc(uport, component_reg_phys, parent_dport);
674 	if (IS_ERR(port))
675 		return port;
676 
677 	dev = &port->dev;
678 	if (is_cxl_memdev(uport))
679 		rc = dev_set_name(dev, "endpoint%d", port->id);
680 	else if (parent_dport)
681 		rc = dev_set_name(dev, "port%d", port->id);
682 	else
683 		rc = dev_set_name(dev, "root%d", port->id);
684 	if (rc)
685 		goto err;
686 
687 	rc = device_add(dev);
688 	if (rc)
689 		goto err;
690 
691 	rc = devm_add_action_or_reset(host, unregister_port, port);
692 	if (rc)
693 		return ERR_PTR(rc);
694 
695 	rc = devm_cxl_link_uport(host, port);
696 	if (rc)
697 		return ERR_PTR(rc);
698 
699 	return port;
700 
701 err:
702 	put_device(dev);
703 	return ERR_PTR(rc);
704 }
705 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_port, CXL);
706 
cxl_port_to_pci_bus(struct cxl_port * port)707 struct pci_bus *cxl_port_to_pci_bus(struct cxl_port *port)
708 {
709 	/* There is no pci_bus associated with a CXL platform-root port */
710 	if (is_cxl_root(port))
711 		return NULL;
712 
713 	if (dev_is_pci(port->uport)) {
714 		struct pci_dev *pdev = to_pci_dev(port->uport);
715 
716 		return pdev->subordinate;
717 	}
718 
719 	return xa_load(&cxl_root_buses, (unsigned long)port->uport);
720 }
721 EXPORT_SYMBOL_NS_GPL(cxl_port_to_pci_bus, CXL);
722 
unregister_pci_bus(void * uport)723 static void unregister_pci_bus(void *uport)
724 {
725 	xa_erase(&cxl_root_buses, (unsigned long)uport);
726 }
727 
devm_cxl_register_pci_bus(struct device * host,struct device * uport,struct pci_bus * bus)728 int devm_cxl_register_pci_bus(struct device *host, struct device *uport,
729 			      struct pci_bus *bus)
730 {
731 	int rc;
732 
733 	if (dev_is_pci(uport))
734 		return -EINVAL;
735 
736 	rc = xa_insert(&cxl_root_buses, (unsigned long)uport, bus, GFP_KERNEL);
737 	if (rc)
738 		return rc;
739 	return devm_add_action_or_reset(host, unregister_pci_bus, uport);
740 }
741 EXPORT_SYMBOL_NS_GPL(devm_cxl_register_pci_bus, CXL);
742 
dev_is_cxl_root_child(struct device * dev)743 static bool dev_is_cxl_root_child(struct device *dev)
744 {
745 	struct cxl_port *port, *parent;
746 
747 	if (!is_cxl_port(dev))
748 		return false;
749 
750 	port = to_cxl_port(dev);
751 	if (is_cxl_root(port))
752 		return false;
753 
754 	parent = to_cxl_port(port->dev.parent);
755 	if (is_cxl_root(parent))
756 		return true;
757 
758 	return false;
759 }
760 
761 /* Find a 2nd level CXL port that has a dport that is an ancestor of @match */
match_root_child(struct device * dev,const void * match)762 static int match_root_child(struct device *dev, const void *match)
763 {
764 	const struct device *iter = NULL;
765 	struct cxl_dport *dport;
766 	struct cxl_port *port;
767 
768 	if (!dev_is_cxl_root_child(dev))
769 		return 0;
770 
771 	port = to_cxl_port(dev);
772 	iter = match;
773 	while (iter) {
774 		dport = cxl_find_dport_by_dev(port, iter);
775 		if (dport)
776 			break;
777 		iter = iter->parent;
778 	}
779 
780 	return !!iter;
781 }
782 
find_cxl_root(struct device * dev)783 struct cxl_port *find_cxl_root(struct device *dev)
784 {
785 	struct device *port_dev;
786 	struct cxl_port *root;
787 
788 	port_dev = bus_find_device(&cxl_bus_type, NULL, dev, match_root_child);
789 	if (!port_dev)
790 		return NULL;
791 
792 	root = to_cxl_port(port_dev->parent);
793 	get_device(&root->dev);
794 	put_device(port_dev);
795 	return root;
796 }
797 EXPORT_SYMBOL_NS_GPL(find_cxl_root, CXL);
798 
find_dport(struct cxl_port * port,int id)799 static struct cxl_dport *find_dport(struct cxl_port *port, int id)
800 {
801 	struct cxl_dport *dport;
802 	unsigned long index;
803 
804 	device_lock_assert(&port->dev);
805 	xa_for_each(&port->dports, index, dport)
806 		if (dport->port_id == id)
807 			return dport;
808 	return NULL;
809 }
810 
add_dport(struct cxl_port * port,struct cxl_dport * new)811 static int add_dport(struct cxl_port *port, struct cxl_dport *new)
812 {
813 	struct cxl_dport *dup;
814 	int rc;
815 
816 	device_lock_assert(&port->dev);
817 	dup = find_dport(port, new->port_id);
818 	if (dup) {
819 		dev_err(&port->dev,
820 			"unable to add dport%d-%s non-unique port id (%s)\n",
821 			new->port_id, dev_name(new->dport),
822 			dev_name(dup->dport));
823 		return -EBUSY;
824 	}
825 
826 	rc = xa_insert(&port->dports, (unsigned long)new->dport, new,
827 		       GFP_KERNEL);
828 	if (rc)
829 		return rc;
830 
831 	port->nr_dports++;
832 	return 0;
833 }
834 
835 /*
836  * Since root-level CXL dports cannot be enumerated by PCI they are not
837  * enumerated by the common port driver that acquires the port lock over
838  * dport add/remove. Instead, root dports are manually added by a
839  * platform driver and cond_cxl_root_lock() is used to take the missing
840  * port lock in that case.
841  */
cond_cxl_root_lock(struct cxl_port * port)842 static void cond_cxl_root_lock(struct cxl_port *port)
843 {
844 	if (is_cxl_root(port))
845 		device_lock(&port->dev);
846 }
847 
cond_cxl_root_unlock(struct cxl_port * port)848 static void cond_cxl_root_unlock(struct cxl_port *port)
849 {
850 	if (is_cxl_root(port))
851 		device_unlock(&port->dev);
852 }
853 
cxl_dport_remove(void * data)854 static void cxl_dport_remove(void *data)
855 {
856 	struct cxl_dport *dport = data;
857 	struct cxl_port *port = dport->port;
858 
859 	xa_erase(&port->dports, (unsigned long) dport->dport);
860 	put_device(dport->dport);
861 }
862 
cxl_dport_unlink(void * data)863 static void cxl_dport_unlink(void *data)
864 {
865 	struct cxl_dport *dport = data;
866 	struct cxl_port *port = dport->port;
867 	char link_name[CXL_TARGET_STRLEN];
868 
869 	sprintf(link_name, "dport%d", dport->port_id);
870 	sysfs_remove_link(&port->dev.kobj, link_name);
871 }
872 
873 /**
874  * devm_cxl_add_dport - append downstream port data to a cxl_port
875  * @port: the cxl_port that references this dport
876  * @dport_dev: firmware or PCI device representing the dport
877  * @port_id: identifier for this dport in a decoder's target list
878  * @component_reg_phys: optional location of CXL component registers
879  *
880  * Note that dports are appended to the devm release action's of the
881  * either the port's host (for root ports), or the port itself (for
882  * switch ports)
883  */
devm_cxl_add_dport(struct cxl_port * port,struct device * dport_dev,int port_id,resource_size_t component_reg_phys)884 struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,
885 				     struct device *dport_dev, int port_id,
886 				     resource_size_t component_reg_phys)
887 {
888 	char link_name[CXL_TARGET_STRLEN];
889 	struct cxl_dport *dport;
890 	struct device *host;
891 	int rc;
892 
893 	if (is_cxl_root(port))
894 		host = port->uport;
895 	else
896 		host = &port->dev;
897 
898 	if (!host->driver) {
899 		dev_WARN_ONCE(&port->dev, 1, "dport:%s bad devm context\n",
900 			      dev_name(dport_dev));
901 		return ERR_PTR(-ENXIO);
902 	}
903 
904 	if (snprintf(link_name, CXL_TARGET_STRLEN, "dport%d", port_id) >=
905 	    CXL_TARGET_STRLEN)
906 		return ERR_PTR(-EINVAL);
907 
908 	dport = devm_kzalloc(host, sizeof(*dport), GFP_KERNEL);
909 	if (!dport)
910 		return ERR_PTR(-ENOMEM);
911 
912 	dport->dport = dport_dev;
913 	dport->port_id = port_id;
914 	dport->component_reg_phys = component_reg_phys;
915 	dport->port = port;
916 
917 	cond_cxl_root_lock(port);
918 	rc = add_dport(port, dport);
919 	cond_cxl_root_unlock(port);
920 	if (rc)
921 		return ERR_PTR(rc);
922 
923 	get_device(dport_dev);
924 	rc = devm_add_action_or_reset(host, cxl_dport_remove, dport);
925 	if (rc)
926 		return ERR_PTR(rc);
927 
928 	rc = sysfs_create_link(&port->dev.kobj, &dport_dev->kobj, link_name);
929 	if (rc)
930 		return ERR_PTR(rc);
931 
932 	rc = devm_add_action_or_reset(host, cxl_dport_unlink, dport);
933 	if (rc)
934 		return ERR_PTR(rc);
935 
936 	return dport;
937 }
938 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport, CXL);
939 
add_ep(struct cxl_ep * new)940 static int add_ep(struct cxl_ep *new)
941 {
942 	struct cxl_port *port = new->dport->port;
943 	int rc;
944 
945 	device_lock(&port->dev);
946 	if (port->dead) {
947 		device_unlock(&port->dev);
948 		return -ENXIO;
949 	}
950 	rc = xa_insert(&port->endpoints, (unsigned long)new->ep, new,
951 		       GFP_KERNEL);
952 	device_unlock(&port->dev);
953 
954 	return rc;
955 }
956 
957 /**
958  * cxl_add_ep - register an endpoint's interest in a port
959  * @dport: the dport that routes to @ep_dev
960  * @ep_dev: device representing the endpoint
961  *
962  * Intermediate CXL ports are scanned based on the arrival of endpoints.
963  * When those endpoints depart the port can be destroyed once all
964  * endpoints that care about that port have been removed.
965  */
cxl_add_ep(struct cxl_dport * dport,struct device * ep_dev)966 static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
967 {
968 	struct cxl_ep *ep;
969 	int rc;
970 
971 	ep = kzalloc(sizeof(*ep), GFP_KERNEL);
972 	if (!ep)
973 		return -ENOMEM;
974 
975 	ep->ep = get_device(ep_dev);
976 	ep->dport = dport;
977 
978 	rc = add_ep(ep);
979 	if (rc)
980 		cxl_ep_release(ep);
981 	return rc;
982 }
983 
984 struct cxl_find_port_ctx {
985 	const struct device *dport_dev;
986 	const struct cxl_port *parent_port;
987 	struct cxl_dport **dport;
988 };
989 
match_port_by_dport(struct device * dev,const void * data)990 static int match_port_by_dport(struct device *dev, const void *data)
991 {
992 	const struct cxl_find_port_ctx *ctx = data;
993 	struct cxl_dport *dport;
994 	struct cxl_port *port;
995 
996 	if (!is_cxl_port(dev))
997 		return 0;
998 	if (ctx->parent_port && dev->parent != &ctx->parent_port->dev)
999 		return 0;
1000 
1001 	port = to_cxl_port(dev);
1002 	dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
1003 	if (ctx->dport)
1004 		*ctx->dport = dport;
1005 	return dport != NULL;
1006 }
1007 
__find_cxl_port(struct cxl_find_port_ctx * ctx)1008 static struct cxl_port *__find_cxl_port(struct cxl_find_port_ctx *ctx)
1009 {
1010 	struct device *dev;
1011 
1012 	if (!ctx->dport_dev)
1013 		return NULL;
1014 
1015 	dev = bus_find_device(&cxl_bus_type, NULL, ctx, match_port_by_dport);
1016 	if (dev)
1017 		return to_cxl_port(dev);
1018 	return NULL;
1019 }
1020 
find_cxl_port(struct device * dport_dev,struct cxl_dport ** dport)1021 static struct cxl_port *find_cxl_port(struct device *dport_dev,
1022 				      struct cxl_dport **dport)
1023 {
1024 	struct cxl_find_port_ctx ctx = {
1025 		.dport_dev = dport_dev,
1026 		.dport = dport,
1027 	};
1028 	struct cxl_port *port;
1029 
1030 	port = __find_cxl_port(&ctx);
1031 	return port;
1032 }
1033 
find_cxl_port_at(struct cxl_port * parent_port,struct device * dport_dev,struct cxl_dport ** dport)1034 static struct cxl_port *find_cxl_port_at(struct cxl_port *parent_port,
1035 					 struct device *dport_dev,
1036 					 struct cxl_dport **dport)
1037 {
1038 	struct cxl_find_port_ctx ctx = {
1039 		.dport_dev = dport_dev,
1040 		.parent_port = parent_port,
1041 		.dport = dport,
1042 	};
1043 	struct cxl_port *port;
1044 
1045 	port = __find_cxl_port(&ctx);
1046 	return port;
1047 }
1048 
1049 /*
1050  * All users of grandparent() are using it to walk PCIe-like swich port
1051  * hierarchy. A PCIe switch is comprised of a bridge device representing the
1052  * upstream switch port and N bridges representing downstream switch ports. When
1053  * bridges stack the grand-parent of a downstream switch port is another
1054  * downstream switch port in the immediate ancestor switch.
1055  */
grandparent(struct device * dev)1056 static struct device *grandparent(struct device *dev)
1057 {
1058 	if (dev && dev->parent)
1059 		return dev->parent->parent;
1060 	return NULL;
1061 }
1062 
delete_endpoint(void * data)1063 static void delete_endpoint(void *data)
1064 {
1065 	struct cxl_memdev *cxlmd = data;
1066 	struct cxl_port *endpoint = dev_get_drvdata(&cxlmd->dev);
1067 	struct cxl_port *parent_port;
1068 	struct device *parent;
1069 
1070 	parent_port = cxl_mem_find_port(cxlmd, NULL);
1071 	if (!parent_port)
1072 		goto out;
1073 	parent = &parent_port->dev;
1074 
1075 	device_lock(parent);
1076 	if (parent->driver && !endpoint->dead) {
1077 		devm_release_action(parent, cxl_unlink_uport, endpoint);
1078 		devm_release_action(parent, unregister_port, endpoint);
1079 	}
1080 	device_unlock(parent);
1081 	put_device(parent);
1082 out:
1083 	put_device(&endpoint->dev);
1084 }
1085 
cxl_endpoint_autoremove(struct cxl_memdev * cxlmd,struct cxl_port * endpoint)1086 int cxl_endpoint_autoremove(struct cxl_memdev *cxlmd, struct cxl_port *endpoint)
1087 {
1088 	struct device *dev = &cxlmd->dev;
1089 
1090 	get_device(&endpoint->dev);
1091 	dev_set_drvdata(dev, endpoint);
1092 	return devm_add_action_or_reset(dev, delete_endpoint, cxlmd);
1093 }
1094 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_autoremove, CXL);
1095 
1096 /*
1097  * The natural end of life of a non-root 'cxl_port' is when its parent port goes
1098  * through a ->remove() event ("top-down" unregistration). The unnatural trigger
1099  * for a port to be unregistered is when all memdevs beneath that port have gone
1100  * through ->remove(). This "bottom-up" removal selectively removes individual
1101  * child ports manually. This depends on devm_cxl_add_port() to not change is
1102  * devm action registration order, and for dports to have already been
1103  * destroyed by reap_dports().
1104  */
delete_switch_port(struct cxl_port * port)1105 static void delete_switch_port(struct cxl_port *port)
1106 {
1107 	devm_release_action(port->dev.parent, cxl_unlink_uport, port);
1108 	devm_release_action(port->dev.parent, unregister_port, port);
1109 }
1110 
reap_dports(struct cxl_port * port)1111 static void reap_dports(struct cxl_port *port)
1112 {
1113 	struct cxl_dport *dport;
1114 	unsigned long index;
1115 
1116 	device_lock_assert(&port->dev);
1117 
1118 	xa_for_each(&port->dports, index, dport) {
1119 		devm_release_action(&port->dev, cxl_dport_unlink, dport);
1120 		devm_release_action(&port->dev, cxl_dport_remove, dport);
1121 		devm_kfree(&port->dev, dport);
1122 	}
1123 }
1124 
devm_cxl_add_endpoint(struct cxl_memdev * cxlmd,struct cxl_dport * parent_dport)1125 int devm_cxl_add_endpoint(struct cxl_memdev *cxlmd,
1126 			  struct cxl_dport *parent_dport)
1127 {
1128 	struct cxl_port *parent_port = parent_dport->port;
1129 	struct cxl_dev_state *cxlds = cxlmd->cxlds;
1130 	struct cxl_port *endpoint, *iter, *down;
1131 	int rc;
1132 
1133 	/*
1134 	 * Now that the path to the root is established record all the
1135 	 * intervening ports in the chain.
1136 	 */
1137 	for (iter = parent_port, down = NULL; !is_cxl_root(iter);
1138 	     down = iter, iter = to_cxl_port(iter->dev.parent)) {
1139 		struct cxl_ep *ep;
1140 
1141 		ep = cxl_ep_load(iter, cxlmd);
1142 		ep->next = down;
1143 	}
1144 
1145 	endpoint = devm_cxl_add_port(&parent_port->dev, &cxlmd->dev,
1146 				     cxlds->component_reg_phys, parent_dport);
1147 	if (IS_ERR(endpoint))
1148 		return PTR_ERR(endpoint);
1149 
1150 	dev_dbg(&cxlmd->dev, "add: %s\n", dev_name(&endpoint->dev));
1151 
1152 	rc = cxl_endpoint_autoremove(cxlmd, endpoint);
1153 	if (rc)
1154 		return rc;
1155 
1156 	if (!endpoint->dev.driver) {
1157 		dev_err(&cxlmd->dev, "%s failed probe\n",
1158 			dev_name(&endpoint->dev));
1159 		return -ENXIO;
1160 	}
1161 
1162 	return 0;
1163 }
1164 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_endpoint, CXL);
1165 
cxl_detach_ep(void * data)1166 static void cxl_detach_ep(void *data)
1167 {
1168 	struct cxl_memdev *cxlmd = data;
1169 	struct device *iter;
1170 
1171 	for (iter = &cxlmd->dev; iter; iter = grandparent(iter)) {
1172 		struct device *dport_dev = grandparent(iter);
1173 		struct cxl_port *port, *parent_port;
1174 		struct cxl_ep *ep;
1175 		bool died = false;
1176 
1177 		if (!dport_dev)
1178 			break;
1179 
1180 		port = find_cxl_port(dport_dev, NULL);
1181 		if (!port)
1182 			continue;
1183 
1184 		if (is_cxl_root(port)) {
1185 			put_device(&port->dev);
1186 			continue;
1187 		}
1188 
1189 		parent_port = to_cxl_port(port->dev.parent);
1190 		device_lock(&parent_port->dev);
1191 		if (!parent_port->dev.driver) {
1192 			/*
1193 			 * The bottom-up race to delete the port lost to a
1194 			 * top-down port disable, give up here, because the
1195 			 * parent_port ->remove() will have cleaned up all
1196 			 * descendants.
1197 			 */
1198 			device_unlock(&parent_port->dev);
1199 			put_device(&port->dev);
1200 			continue;
1201 		}
1202 
1203 		device_lock(&port->dev);
1204 		ep = cxl_ep_load(port, cxlmd);
1205 		dev_dbg(&cxlmd->dev, "disconnect %s from %s\n",
1206 			ep ? dev_name(ep->ep) : "", dev_name(&port->dev));
1207 		cxl_ep_remove(port, ep);
1208 		if (ep && !port->dead && xa_empty(&port->endpoints) &&
1209 		    !is_cxl_root(parent_port)) {
1210 			/*
1211 			 * This was the last ep attached to a dynamically
1212 			 * enumerated port. Block new cxl_add_ep() and garbage
1213 			 * collect the port.
1214 			 */
1215 			died = true;
1216 			port->dead = true;
1217 			reap_dports(port);
1218 		}
1219 		device_unlock(&port->dev);
1220 
1221 		if (died) {
1222 			dev_dbg(&cxlmd->dev, "delete %s\n",
1223 				dev_name(&port->dev));
1224 			delete_switch_port(port);
1225 		}
1226 		put_device(&port->dev);
1227 		device_unlock(&parent_port->dev);
1228 	}
1229 }
1230 
find_component_registers(struct device * dev)1231 static resource_size_t find_component_registers(struct device *dev)
1232 {
1233 	struct cxl_register_map map;
1234 	struct pci_dev *pdev;
1235 
1236 	/*
1237 	 * Theoretically, CXL component registers can be hosted on a
1238 	 * non-PCI device, in practice, only cxl_test hits this case.
1239 	 */
1240 	if (!dev_is_pci(dev))
1241 		return CXL_RESOURCE_NONE;
1242 
1243 	pdev = to_pci_dev(dev);
1244 
1245 	cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map);
1246 	return cxl_regmap_to_base(pdev, &map);
1247 }
1248 
add_port_attach_ep(struct cxl_memdev * cxlmd,struct device * uport_dev,struct device * dport_dev)1249 static int add_port_attach_ep(struct cxl_memdev *cxlmd,
1250 			      struct device *uport_dev,
1251 			      struct device *dport_dev)
1252 {
1253 	struct device *dparent = grandparent(dport_dev);
1254 	struct cxl_port *port, *parent_port = NULL;
1255 	struct cxl_dport *dport, *parent_dport;
1256 	resource_size_t component_reg_phys;
1257 	int rc;
1258 
1259 	if (!dparent) {
1260 		/*
1261 		 * The iteration reached the topology root without finding the
1262 		 * CXL-root 'cxl_port' on a previous iteration, fail for now to
1263 		 * be re-probed after platform driver attaches.
1264 		 */
1265 		dev_dbg(&cxlmd->dev, "%s is a root dport\n",
1266 			dev_name(dport_dev));
1267 		return -ENXIO;
1268 	}
1269 
1270 	parent_port = find_cxl_port(dparent, &parent_dport);
1271 	if (!parent_port) {
1272 		/* iterate to create this parent_port */
1273 		return -EAGAIN;
1274 	}
1275 
1276 	device_lock(&parent_port->dev);
1277 	if (!parent_port->dev.driver) {
1278 		dev_warn(&cxlmd->dev,
1279 			 "port %s:%s disabled, failed to enumerate CXL.mem\n",
1280 			 dev_name(&parent_port->dev), dev_name(uport_dev));
1281 		port = ERR_PTR(-ENXIO);
1282 		goto out;
1283 	}
1284 
1285 	port = find_cxl_port_at(parent_port, dport_dev, &dport);
1286 	if (!port) {
1287 		component_reg_phys = find_component_registers(uport_dev);
1288 		port = devm_cxl_add_port(&parent_port->dev, uport_dev,
1289 					 component_reg_phys, parent_dport);
1290 		/* retry find to pick up the new dport information */
1291 		if (!IS_ERR(port))
1292 			port = find_cxl_port_at(parent_port, dport_dev, &dport);
1293 	}
1294 out:
1295 	device_unlock(&parent_port->dev);
1296 
1297 	if (IS_ERR(port))
1298 		rc = PTR_ERR(port);
1299 	else {
1300 		dev_dbg(&cxlmd->dev, "add to new port %s:%s\n",
1301 			dev_name(&port->dev), dev_name(port->uport));
1302 		rc = cxl_add_ep(dport, &cxlmd->dev);
1303 		if (rc == -EBUSY) {
1304 			/*
1305 			 * "can't" happen, but this error code means
1306 			 * something to the caller, so translate it.
1307 			 */
1308 			rc = -ENXIO;
1309 		}
1310 		put_device(&port->dev);
1311 	}
1312 
1313 	put_device(&parent_port->dev);
1314 	return rc;
1315 }
1316 
devm_cxl_enumerate_ports(struct cxl_memdev * cxlmd)1317 int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
1318 {
1319 	struct device *dev = &cxlmd->dev;
1320 	struct device *iter;
1321 	int rc;
1322 
1323 	rc = devm_add_action_or_reset(&cxlmd->dev, cxl_detach_ep, cxlmd);
1324 	if (rc)
1325 		return rc;
1326 
1327 	/*
1328 	 * Scan for and add all cxl_ports in this device's ancestry.
1329 	 * Repeat until no more ports are added. Abort if a port add
1330 	 * attempt fails.
1331 	 */
1332 retry:
1333 	for (iter = dev; iter; iter = grandparent(iter)) {
1334 		struct device *dport_dev = grandparent(iter);
1335 		struct device *uport_dev;
1336 		struct cxl_dport *dport;
1337 		struct cxl_port *port;
1338 
1339 		if (!dport_dev)
1340 			return 0;
1341 
1342 		uport_dev = dport_dev->parent;
1343 		if (!uport_dev) {
1344 			dev_warn(dev, "at %s no parent for dport: %s\n",
1345 				 dev_name(iter), dev_name(dport_dev));
1346 			return -ENXIO;
1347 		}
1348 
1349 		dev_dbg(dev, "scan: iter: %s dport_dev: %s parent: %s\n",
1350 			dev_name(iter), dev_name(dport_dev),
1351 			dev_name(uport_dev));
1352 		port = find_cxl_port(dport_dev, &dport);
1353 		if (port) {
1354 			dev_dbg(&cxlmd->dev,
1355 				"found already registered port %s:%s\n",
1356 				dev_name(&port->dev), dev_name(port->uport));
1357 			rc = cxl_add_ep(dport, &cxlmd->dev);
1358 
1359 			/*
1360 			 * If the endpoint already exists in the port's list,
1361 			 * that's ok, it was added on a previous pass.
1362 			 * Otherwise, retry in add_port_attach_ep() after taking
1363 			 * the parent_port lock as the current port may be being
1364 			 * reaped.
1365 			 */
1366 			if (rc && rc != -EBUSY) {
1367 				put_device(&port->dev);
1368 				return rc;
1369 			}
1370 
1371 			/* Any more ports to add between this one and the root? */
1372 			if (!dev_is_cxl_root_child(&port->dev)) {
1373 				put_device(&port->dev);
1374 				continue;
1375 			}
1376 
1377 			put_device(&port->dev);
1378 			return 0;
1379 		}
1380 
1381 		rc = add_port_attach_ep(cxlmd, uport_dev, dport_dev);
1382 		/* port missing, try to add parent */
1383 		if (rc == -EAGAIN)
1384 			continue;
1385 		/* failed to add ep or port */
1386 		if (rc)
1387 			return rc;
1388 		/* port added, new descendants possible, start over */
1389 		goto retry;
1390 	}
1391 
1392 	return 0;
1393 }
1394 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, CXL);
1395 
cxl_mem_find_port(struct cxl_memdev * cxlmd,struct cxl_dport ** dport)1396 struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
1397 				   struct cxl_dport **dport)
1398 {
1399 	return find_cxl_port(grandparent(&cxlmd->dev), dport);
1400 }
1401 EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, CXL);
1402 
decoder_populate_targets(struct cxl_switch_decoder * cxlsd,struct cxl_port * port,int * target_map)1403 static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
1404 				    struct cxl_port *port, int *target_map)
1405 {
1406 	int i, rc = 0;
1407 
1408 	if (!target_map)
1409 		return 0;
1410 
1411 	device_lock_assert(&port->dev);
1412 
1413 	if (xa_empty(&port->dports))
1414 		return -EINVAL;
1415 
1416 	write_seqlock(&cxlsd->target_lock);
1417 	for (i = 0; i < cxlsd->nr_targets; i++) {
1418 		struct cxl_dport *dport = find_dport(port, target_map[i]);
1419 
1420 		if (!dport) {
1421 			rc = -ENXIO;
1422 			break;
1423 		}
1424 		cxlsd->target[i] = dport;
1425 	}
1426 	write_sequnlock(&cxlsd->target_lock);
1427 
1428 	return rc;
1429 }
1430 
cxl_hb_modulo(struct cxl_root_decoder * cxlrd,int pos)1431 static struct cxl_dport *cxl_hb_modulo(struct cxl_root_decoder *cxlrd, int pos)
1432 {
1433 	struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd;
1434 	struct cxl_decoder *cxld = &cxlsd->cxld;
1435 	int iw;
1436 
1437 	iw = cxld->interleave_ways;
1438 	if (dev_WARN_ONCE(&cxld->dev, iw != cxlsd->nr_targets,
1439 			  "misconfigured root decoder\n"))
1440 		return NULL;
1441 
1442 	return cxlrd->cxlsd.target[pos % iw];
1443 }
1444 
1445 static struct lock_class_key cxl_decoder_key;
1446 
1447 /**
1448  * cxl_decoder_init - Common decoder setup / initialization
1449  * @port: owning port of this decoder
1450  * @cxld: common decoder properties to initialize
1451  *
1452  * A port may contain one or more decoders. Each of those decoders
1453  * enable some address space for CXL.mem utilization. A decoder is
1454  * expected to be configured by the caller before registering via
1455  * cxl_decoder_add()
1456  */
cxl_decoder_init(struct cxl_port * port,struct cxl_decoder * cxld)1457 static int cxl_decoder_init(struct cxl_port *port, struct cxl_decoder *cxld)
1458 {
1459 	struct device *dev;
1460 	int rc;
1461 
1462 	rc = ida_alloc(&port->decoder_ida, GFP_KERNEL);
1463 	if (rc < 0)
1464 		return rc;
1465 
1466 	/* need parent to stick around to release the id */
1467 	get_device(&port->dev);
1468 	cxld->id = rc;
1469 
1470 	dev = &cxld->dev;
1471 	device_initialize(dev);
1472 	lockdep_set_class(&dev->mutex, &cxl_decoder_key);
1473 	device_set_pm_not_required(dev);
1474 	dev->parent = &port->dev;
1475 	dev->bus = &cxl_bus_type;
1476 
1477 	/* Pre initialize an "empty" decoder */
1478 	cxld->interleave_ways = 1;
1479 	cxld->interleave_granularity = PAGE_SIZE;
1480 	cxld->target_type = CXL_DECODER_EXPANDER;
1481 	cxld->hpa_range = (struct range) {
1482 		.start = 0,
1483 		.end = -1,
1484 	};
1485 
1486 	return 0;
1487 }
1488 
cxl_switch_decoder_init(struct cxl_port * port,struct cxl_switch_decoder * cxlsd,int nr_targets)1489 static int cxl_switch_decoder_init(struct cxl_port *port,
1490 				   struct cxl_switch_decoder *cxlsd,
1491 				   int nr_targets)
1492 {
1493 	if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
1494 		return -EINVAL;
1495 
1496 	cxlsd->nr_targets = nr_targets;
1497 	seqlock_init(&cxlsd->target_lock);
1498 	return cxl_decoder_init(port, &cxlsd->cxld);
1499 }
1500 
1501 /**
1502  * cxl_root_decoder_alloc - Allocate a root level decoder
1503  * @port: owning CXL root of this decoder
1504  * @nr_targets: static number of downstream targets
1505  *
1506  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1507  * 'CXL root' decoder is one that decodes from a top-level / static platform
1508  * firmware description of CXL resources into a CXL standard decode
1509  * topology.
1510  */
cxl_root_decoder_alloc(struct cxl_port * port,unsigned int nr_targets)1511 struct cxl_root_decoder *cxl_root_decoder_alloc(struct cxl_port *port,
1512 						unsigned int nr_targets)
1513 {
1514 	struct cxl_root_decoder *cxlrd;
1515 	struct cxl_switch_decoder *cxlsd;
1516 	struct cxl_decoder *cxld;
1517 	int rc;
1518 
1519 	if (!is_cxl_root(port))
1520 		return ERR_PTR(-EINVAL);
1521 
1522 	cxlrd = kzalloc(struct_size(cxlrd, cxlsd.target, nr_targets),
1523 			GFP_KERNEL);
1524 	if (!cxlrd)
1525 		return ERR_PTR(-ENOMEM);
1526 
1527 	cxlsd = &cxlrd->cxlsd;
1528 	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1529 	if (rc) {
1530 		kfree(cxlrd);
1531 		return ERR_PTR(rc);
1532 	}
1533 
1534 	cxlrd->calc_hb = cxl_hb_modulo;
1535 
1536 	cxld = &cxlsd->cxld;
1537 	cxld->dev.type = &cxl_decoder_root_type;
1538 	/*
1539 	 * cxl_root_decoder_release() special cases negative ids to
1540 	 * detect memregion_alloc() failures.
1541 	 */
1542 	atomic_set(&cxlrd->region_id, -1);
1543 	rc = memregion_alloc(GFP_KERNEL);
1544 	if (rc < 0) {
1545 		put_device(&cxld->dev);
1546 		return ERR_PTR(rc);
1547 	}
1548 
1549 	atomic_set(&cxlrd->region_id, rc);
1550 	return cxlrd;
1551 }
1552 EXPORT_SYMBOL_NS_GPL(cxl_root_decoder_alloc, CXL);
1553 
1554 /**
1555  * cxl_switch_decoder_alloc - Allocate a switch level decoder
1556  * @port: owning CXL switch port of this decoder
1557  * @nr_targets: max number of dynamically addressable downstream targets
1558  *
1559  * Return: A new cxl decoder to be registered by cxl_decoder_add(). A
1560  * 'switch' decoder is any decoder that can be enumerated by PCIe
1561  * topology and the HDM Decoder Capability. This includes the decoders
1562  * that sit between Switch Upstream Ports / Switch Downstream Ports and
1563  * Host Bridges / Root Ports.
1564  */
cxl_switch_decoder_alloc(struct cxl_port * port,unsigned int nr_targets)1565 struct cxl_switch_decoder *cxl_switch_decoder_alloc(struct cxl_port *port,
1566 						    unsigned int nr_targets)
1567 {
1568 	struct cxl_switch_decoder *cxlsd;
1569 	struct cxl_decoder *cxld;
1570 	int rc;
1571 
1572 	if (is_cxl_root(port) || is_cxl_endpoint(port))
1573 		return ERR_PTR(-EINVAL);
1574 
1575 	cxlsd = kzalloc(struct_size(cxlsd, target, nr_targets), GFP_KERNEL);
1576 	if (!cxlsd)
1577 		return ERR_PTR(-ENOMEM);
1578 
1579 	rc = cxl_switch_decoder_init(port, cxlsd, nr_targets);
1580 	if (rc) {
1581 		kfree(cxlsd);
1582 		return ERR_PTR(rc);
1583 	}
1584 
1585 	cxld = &cxlsd->cxld;
1586 	cxld->dev.type = &cxl_decoder_switch_type;
1587 	return cxlsd;
1588 }
1589 EXPORT_SYMBOL_NS_GPL(cxl_switch_decoder_alloc, CXL);
1590 
1591 /**
1592  * cxl_endpoint_decoder_alloc - Allocate an endpoint decoder
1593  * @port: owning port of this decoder
1594  *
1595  * Return: A new cxl decoder to be registered by cxl_decoder_add()
1596  */
cxl_endpoint_decoder_alloc(struct cxl_port * port)1597 struct cxl_endpoint_decoder *cxl_endpoint_decoder_alloc(struct cxl_port *port)
1598 {
1599 	struct cxl_endpoint_decoder *cxled;
1600 	struct cxl_decoder *cxld;
1601 	int rc;
1602 
1603 	if (!is_cxl_endpoint(port))
1604 		return ERR_PTR(-EINVAL);
1605 
1606 	cxled = kzalloc(sizeof(*cxled), GFP_KERNEL);
1607 	if (!cxled)
1608 		return ERR_PTR(-ENOMEM);
1609 
1610 	cxled->pos = -1;
1611 	cxld = &cxled->cxld;
1612 	rc = cxl_decoder_init(port, cxld);
1613 	if (rc)	 {
1614 		kfree(cxled);
1615 		return ERR_PTR(rc);
1616 	}
1617 
1618 	cxld->dev.type = &cxl_decoder_endpoint_type;
1619 	return cxled;
1620 }
1621 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_alloc, CXL);
1622 
1623 /**
1624  * cxl_decoder_add_locked - Add a decoder with targets
1625  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1626  * @target_map: A list of downstream ports that this decoder can direct memory
1627  *              traffic to. These numbers should correspond with the port number
1628  *              in the PCIe Link Capabilities structure.
1629  *
1630  * Certain types of decoders may not have any targets. The main example of this
1631  * is an endpoint device. A more awkward example is a hostbridge whose root
1632  * ports get hot added (technically possible, though unlikely).
1633  *
1634  * This is the locked variant of cxl_decoder_add().
1635  *
1636  * Context: Process context. Expects the device lock of the port that owns the
1637  *	    @cxld to be held.
1638  *
1639  * Return: Negative error code if the decoder wasn't properly configured; else
1640  *	   returns 0.
1641  */
cxl_decoder_add_locked(struct cxl_decoder * cxld,int * target_map)1642 int cxl_decoder_add_locked(struct cxl_decoder *cxld, int *target_map)
1643 {
1644 	struct cxl_port *port;
1645 	struct device *dev;
1646 	int rc;
1647 
1648 	if (WARN_ON_ONCE(!cxld))
1649 		return -EINVAL;
1650 
1651 	if (WARN_ON_ONCE(IS_ERR(cxld)))
1652 		return PTR_ERR(cxld);
1653 
1654 	if (cxld->interleave_ways < 1)
1655 		return -EINVAL;
1656 
1657 	dev = &cxld->dev;
1658 
1659 	port = to_cxl_port(cxld->dev.parent);
1660 	if (!is_endpoint_decoder(dev)) {
1661 		struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(dev);
1662 
1663 		rc = decoder_populate_targets(cxlsd, port, target_map);
1664 		if (rc && (cxld->flags & CXL_DECODER_F_ENABLE)) {
1665 			dev_err(&port->dev,
1666 				"Failed to populate active decoder targets\n");
1667 			return rc;
1668 		}
1669 	}
1670 
1671 	rc = dev_set_name(dev, "decoder%d.%d", port->id, cxld->id);
1672 	if (rc)
1673 		return rc;
1674 
1675 	return device_add(dev);
1676 }
1677 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add_locked, CXL);
1678 
1679 /**
1680  * cxl_decoder_add - Add a decoder with targets
1681  * @cxld: The cxl decoder allocated by cxl_<type>_decoder_alloc()
1682  * @target_map: A list of downstream ports that this decoder can direct memory
1683  *              traffic to. These numbers should correspond with the port number
1684  *              in the PCIe Link Capabilities structure.
1685  *
1686  * This is the unlocked variant of cxl_decoder_add_locked().
1687  * See cxl_decoder_add_locked().
1688  *
1689  * Context: Process context. Takes and releases the device lock of the port that
1690  *	    owns the @cxld.
1691  */
cxl_decoder_add(struct cxl_decoder * cxld,int * target_map)1692 int cxl_decoder_add(struct cxl_decoder *cxld, int *target_map)
1693 {
1694 	struct cxl_port *port;
1695 	int rc;
1696 
1697 	if (WARN_ON_ONCE(!cxld))
1698 		return -EINVAL;
1699 
1700 	if (WARN_ON_ONCE(IS_ERR(cxld)))
1701 		return PTR_ERR(cxld);
1702 
1703 	port = to_cxl_port(cxld->dev.parent);
1704 
1705 	device_lock(&port->dev);
1706 	rc = cxl_decoder_add_locked(cxld, target_map);
1707 	device_unlock(&port->dev);
1708 
1709 	return rc;
1710 }
1711 EXPORT_SYMBOL_NS_GPL(cxl_decoder_add, CXL);
1712 
cxld_unregister(void * dev)1713 static void cxld_unregister(void *dev)
1714 {
1715 	struct cxl_endpoint_decoder *cxled;
1716 
1717 	if (is_endpoint_decoder(dev)) {
1718 		cxled = to_cxl_endpoint_decoder(dev);
1719 		cxl_decoder_kill_region(cxled);
1720 	}
1721 
1722 	device_unregister(dev);
1723 }
1724 
cxl_decoder_autoremove(struct device * host,struct cxl_decoder * cxld)1725 int cxl_decoder_autoremove(struct device *host, struct cxl_decoder *cxld)
1726 {
1727 	return devm_add_action_or_reset(host, cxld_unregister, &cxld->dev);
1728 }
1729 EXPORT_SYMBOL_NS_GPL(cxl_decoder_autoremove, CXL);
1730 
1731 /**
1732  * __cxl_driver_register - register a driver for the cxl bus
1733  * @cxl_drv: cxl driver structure to attach
1734  * @owner: owning module/driver
1735  * @modname: KBUILD_MODNAME for parent driver
1736  */
__cxl_driver_register(struct cxl_driver * cxl_drv,struct module * owner,const char * modname)1737 int __cxl_driver_register(struct cxl_driver *cxl_drv, struct module *owner,
1738 			  const char *modname)
1739 {
1740 	if (!cxl_drv->probe) {
1741 		pr_debug("%s ->probe() must be specified\n", modname);
1742 		return -EINVAL;
1743 	}
1744 
1745 	if (!cxl_drv->name) {
1746 		pr_debug("%s ->name must be specified\n", modname);
1747 		return -EINVAL;
1748 	}
1749 
1750 	if (!cxl_drv->id) {
1751 		pr_debug("%s ->id must be specified\n", modname);
1752 		return -EINVAL;
1753 	}
1754 
1755 	cxl_drv->drv.bus = &cxl_bus_type;
1756 	cxl_drv->drv.owner = owner;
1757 	cxl_drv->drv.mod_name = modname;
1758 	cxl_drv->drv.name = cxl_drv->name;
1759 
1760 	return driver_register(&cxl_drv->drv);
1761 }
1762 EXPORT_SYMBOL_NS_GPL(__cxl_driver_register, CXL);
1763 
cxl_driver_unregister(struct cxl_driver * cxl_drv)1764 void cxl_driver_unregister(struct cxl_driver *cxl_drv)
1765 {
1766 	driver_unregister(&cxl_drv->drv);
1767 }
1768 EXPORT_SYMBOL_NS_GPL(cxl_driver_unregister, CXL);
1769 
cxl_bus_uevent(struct device * dev,struct kobj_uevent_env * env)1770 static int cxl_bus_uevent(struct device *dev, struct kobj_uevent_env *env)
1771 {
1772 	return add_uevent_var(env, "MODALIAS=" CXL_MODALIAS_FMT,
1773 			      cxl_device_id(dev));
1774 }
1775 
cxl_bus_match(struct device * dev,struct device_driver * drv)1776 static int cxl_bus_match(struct device *dev, struct device_driver *drv)
1777 {
1778 	return cxl_device_id(dev) == to_cxl_drv(drv)->id;
1779 }
1780 
cxl_bus_probe(struct device * dev)1781 static int cxl_bus_probe(struct device *dev)
1782 {
1783 	int rc;
1784 
1785 	rc = to_cxl_drv(dev->driver)->probe(dev);
1786 	dev_dbg(dev, "probe: %d\n", rc);
1787 	return rc;
1788 }
1789 
cxl_bus_remove(struct device * dev)1790 static void cxl_bus_remove(struct device *dev)
1791 {
1792 	struct cxl_driver *cxl_drv = to_cxl_drv(dev->driver);
1793 
1794 	if (cxl_drv->remove)
1795 		cxl_drv->remove(dev);
1796 }
1797 
1798 static struct workqueue_struct *cxl_bus_wq;
1799 
cxl_bus_rescan(void)1800 int cxl_bus_rescan(void)
1801 {
1802 	return bus_rescan_devices(&cxl_bus_type);
1803 }
1804 EXPORT_SYMBOL_NS_GPL(cxl_bus_rescan, CXL);
1805 
schedule_cxl_memdev_detach(struct cxl_memdev * cxlmd)1806 bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd)
1807 {
1808 	return queue_work(cxl_bus_wq, &cxlmd->detach_work);
1809 }
1810 EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, CXL);
1811 
1812 /* for user tooling to ensure port disable work has completed */
flush_store(struct bus_type * bus,const char * buf,size_t count)1813 static ssize_t flush_store(struct bus_type *bus, const char *buf, size_t count)
1814 {
1815 	if (sysfs_streq(buf, "1")) {
1816 		flush_workqueue(cxl_bus_wq);
1817 		return count;
1818 	}
1819 
1820 	return -EINVAL;
1821 }
1822 
1823 static BUS_ATTR_WO(flush);
1824 
1825 static struct attribute *cxl_bus_attributes[] = {
1826 	&bus_attr_flush.attr,
1827 	NULL,
1828 };
1829 
1830 static struct attribute_group cxl_bus_attribute_group = {
1831 	.attrs = cxl_bus_attributes,
1832 };
1833 
1834 static const struct attribute_group *cxl_bus_attribute_groups[] = {
1835 	&cxl_bus_attribute_group,
1836 	NULL,
1837 };
1838 
1839 struct bus_type cxl_bus_type = {
1840 	.name = "cxl",
1841 	.uevent = cxl_bus_uevent,
1842 	.match = cxl_bus_match,
1843 	.probe = cxl_bus_probe,
1844 	.remove = cxl_bus_remove,
1845 	.bus_groups = cxl_bus_attribute_groups,
1846 };
1847 EXPORT_SYMBOL_NS_GPL(cxl_bus_type, CXL);
1848 
1849 static struct dentry *cxl_debugfs;
1850 
cxl_debugfs_create_dir(const char * dir)1851 struct dentry *cxl_debugfs_create_dir(const char *dir)
1852 {
1853 	return debugfs_create_dir(dir, cxl_debugfs);
1854 }
1855 EXPORT_SYMBOL_NS_GPL(cxl_debugfs_create_dir, CXL);
1856 
cxl_core_init(void)1857 static __init int cxl_core_init(void)
1858 {
1859 	int rc;
1860 
1861 	cxl_debugfs = debugfs_create_dir("cxl", NULL);
1862 
1863 	cxl_mbox_init();
1864 
1865 	rc = cxl_memdev_init();
1866 	if (rc)
1867 		return rc;
1868 
1869 	cxl_bus_wq = alloc_ordered_workqueue("cxl_port", 0);
1870 	if (!cxl_bus_wq) {
1871 		rc = -ENOMEM;
1872 		goto err_wq;
1873 	}
1874 
1875 	rc = bus_register(&cxl_bus_type);
1876 	if (rc)
1877 		goto err_bus;
1878 
1879 	rc = cxl_region_init();
1880 	if (rc)
1881 		goto err_region;
1882 
1883 	return 0;
1884 
1885 err_region:
1886 	bus_unregister(&cxl_bus_type);
1887 err_bus:
1888 	destroy_workqueue(cxl_bus_wq);
1889 err_wq:
1890 	cxl_memdev_exit();
1891 	return rc;
1892 }
1893 
cxl_core_exit(void)1894 static void cxl_core_exit(void)
1895 {
1896 	cxl_region_exit();
1897 	bus_unregister(&cxl_bus_type);
1898 	destroy_workqueue(cxl_bus_wq);
1899 	cxl_memdev_exit();
1900 	debugfs_remove_recursive(cxl_debugfs);
1901 }
1902 
1903 module_init(cxl_core_init);
1904 module_exit(cxl_core_exit);
1905 MODULE_LICENSE("GPL v2");
1906