1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2020 Intel Corporation. */
3 #include <linux/io-64-nonatomic-lo-hi.h>
4 #include <linux/device.h>
5 #include <linux/slab.h>
6 #include <linux/pci.h>
7 #include <cxlmem.h>
8 #include <cxlpci.h>
9 #include <pmu.h>
10 
11 #include "core.h"
12 
13 /**
14  * DOC: cxl registers
15  *
16  * CXL device capabilities are enumerated by PCI DVSEC (Designated
17  * Vendor-specific) and / or descriptors provided by platform firmware.
18  * They can be defined as a set like the device and component registers
19  * mandated by CXL Section 8.1.12.2 Memory Device PCIe Capabilities and
20  * Extended Capabilities, or they can be individual capabilities
21  * appended to bridged and endpoint devices.
22  *
23  * Provide common infrastructure for enumerating and mapping these
24  * discrete capabilities.
25  */
26 
27 /**
28  * cxl_probe_component_regs() - Detect CXL Component register blocks
29  * @dev: Host device of the @base mapping
30  * @base: Mapping containing the HDM Decoder Capability Header
31  * @map: Map object describing the register block information found
32  *
33  * See CXL 2.0 8.2.4 Component Register Layout and Definition
34  * See CXL 2.0 8.2.5.5 CXL Device Register Interface
35  *
36  * Probe for component register information and return it in map object.
37  */
cxl_probe_component_regs(struct device * dev,void __iomem * base,struct cxl_component_reg_map * map)38 void cxl_probe_component_regs(struct device *dev, void __iomem *base,
39 			      struct cxl_component_reg_map *map)
40 {
41 	int cap, cap_count;
42 	u32 cap_array;
43 
44 	*map = (struct cxl_component_reg_map) { 0 };
45 
46 	/*
47 	 * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in
48 	 * CXL 2.0 8.2.4 Table 141.
49 	 */
50 	base += CXL_CM_OFFSET;
51 
52 	cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET);
53 
54 	if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) {
55 		dev_err(dev,
56 			"Couldn't locate the CXL.cache and CXL.mem capability array header.\n");
57 		return;
58 	}
59 
60 	/* It's assumed that future versions will be backward compatible */
61 	cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array);
62 
63 	for (cap = 1; cap <= cap_count; cap++) {
64 		void __iomem *register_block;
65 		struct cxl_reg_map *rmap;
66 		u16 cap_id, offset;
67 		u32 length, hdr;
68 
69 		hdr = readl(base + cap * 0x4);
70 
71 		cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr);
72 		offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr);
73 		register_block = base + offset;
74 		hdr = readl(register_block);
75 
76 		rmap = NULL;
77 		switch (cap_id) {
78 		case CXL_CM_CAP_CAP_ID_HDM: {
79 			int decoder_cnt;
80 
81 			dev_dbg(dev, "found HDM decoder capability (0x%x)\n",
82 				offset);
83 
84 			decoder_cnt = cxl_hdm_decoder_count(hdr);
85 			length = 0x20 * decoder_cnt + 0x10;
86 			rmap = &map->hdm_decoder;
87 			break;
88 		}
89 		case CXL_CM_CAP_CAP_ID_RAS:
90 			dev_dbg(dev, "found RAS capability (0x%x)\n",
91 				offset);
92 			length = CXL_RAS_CAPABILITY_LENGTH;
93 			rmap = &map->ras;
94 			break;
95 		default:
96 			dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id,
97 				offset);
98 			break;
99 		}
100 
101 		if (!rmap)
102 			continue;
103 		rmap->valid = true;
104 		rmap->id = cap_id;
105 		rmap->offset = CXL_CM_OFFSET + offset;
106 		rmap->size = length;
107 	}
108 }
109 EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL);
110 
111 /**
112  * cxl_probe_device_regs() - Detect CXL Device register blocks
113  * @dev: Host device of the @base mapping
114  * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface
115  * @map: Map object describing the register block information found
116  *
117  * Probe for device register information and return it in map object.
118  */
cxl_probe_device_regs(struct device * dev,void __iomem * base,struct cxl_device_reg_map * map)119 void cxl_probe_device_regs(struct device *dev, void __iomem *base,
120 			   struct cxl_device_reg_map *map)
121 {
122 	int cap, cap_count;
123 	u64 cap_array;
124 
125 	*map = (struct cxl_device_reg_map){ 0 };
126 
127 	cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET);
128 	if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) !=
129 	    CXLDEV_CAP_ARRAY_CAP_ID)
130 		return;
131 
132 	cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array);
133 
134 	for (cap = 1; cap <= cap_count; cap++) {
135 		struct cxl_reg_map *rmap;
136 		u32 offset, length;
137 		u16 cap_id;
138 
139 		cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK,
140 				   readl(base + cap * 0x10));
141 		offset = readl(base + cap * 0x10 + 0x4);
142 		length = readl(base + cap * 0x10 + 0x8);
143 
144 		rmap = NULL;
145 		switch (cap_id) {
146 		case CXLDEV_CAP_CAP_ID_DEVICE_STATUS:
147 			dev_dbg(dev, "found Status capability (0x%x)\n", offset);
148 			rmap = &map->status;
149 			break;
150 		case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX:
151 			dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset);
152 			rmap = &map->mbox;
153 			break;
154 		case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX:
155 			dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset);
156 			break;
157 		case CXLDEV_CAP_CAP_ID_MEMDEV:
158 			dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset);
159 			rmap = &map->memdev;
160 			break;
161 		default:
162 			if (cap_id >= 0x8000)
163 				dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset);
164 			else
165 				dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset);
166 			break;
167 		}
168 
169 		if (!rmap)
170 			continue;
171 		rmap->valid = true;
172 		rmap->id = cap_id;
173 		rmap->offset = offset;
174 		rmap->size = length;
175 	}
176 }
177 EXPORT_SYMBOL_NS_GPL(cxl_probe_device_regs, CXL);
178 
devm_cxl_iomap_block(struct device * dev,resource_size_t addr,resource_size_t length)179 void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr,
180 				   resource_size_t length)
181 {
182 	void __iomem *ret_val;
183 	struct resource *res;
184 
185 	if (WARN_ON_ONCE(addr == CXL_RESOURCE_NONE))
186 		return NULL;
187 
188 	res = devm_request_mem_region(dev, addr, length, dev_name(dev));
189 	if (!res) {
190 		resource_size_t end = addr + length - 1;
191 
192 		dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end);
193 		return NULL;
194 	}
195 
196 	ret_val = devm_ioremap(dev, addr, length);
197 	if (!ret_val)
198 		dev_err(dev, "Failed to map region %pr\n", res);
199 
200 	return ret_val;
201 }
202 
cxl_map_component_regs(const struct cxl_register_map * map,struct cxl_component_regs * regs,unsigned long map_mask)203 int cxl_map_component_regs(const struct cxl_register_map *map,
204 			   struct cxl_component_regs *regs,
205 			   unsigned long map_mask)
206 {
207 	struct device *host = map->host;
208 	struct mapinfo {
209 		const struct cxl_reg_map *rmap;
210 		void __iomem **addr;
211 	} mapinfo[] = {
212 		{ &map->component_map.hdm_decoder, &regs->hdm_decoder },
213 		{ &map->component_map.ras, &regs->ras },
214 	};
215 	int i;
216 
217 	for (i = 0; i < ARRAY_SIZE(mapinfo); i++) {
218 		struct mapinfo *mi = &mapinfo[i];
219 		resource_size_t phys_addr;
220 		resource_size_t length;
221 
222 		if (!mi->rmap->valid)
223 			continue;
224 		if (!test_bit(mi->rmap->id, &map_mask))
225 			continue;
226 		phys_addr = map->resource + mi->rmap->offset;
227 		length = mi->rmap->size;
228 		*(mi->addr) = devm_cxl_iomap_block(host, phys_addr, length);
229 		if (!*(mi->addr))
230 			return -ENOMEM;
231 	}
232 
233 	return 0;
234 }
235 EXPORT_SYMBOL_NS_GPL(cxl_map_component_regs, CXL);
236 
cxl_map_device_regs(const struct cxl_register_map * map,struct cxl_device_regs * regs)237 int cxl_map_device_regs(const struct cxl_register_map *map,
238 			struct cxl_device_regs *regs)
239 {
240 	struct device *host = map->host;
241 	resource_size_t phys_addr = map->resource;
242 	struct mapinfo {
243 		const struct cxl_reg_map *rmap;
244 		void __iomem **addr;
245 	} mapinfo[] = {
246 		{ &map->device_map.status, &regs->status, },
247 		{ &map->device_map.mbox, &regs->mbox, },
248 		{ &map->device_map.memdev, &regs->memdev, },
249 	};
250 	int i;
251 
252 	for (i = 0; i < ARRAY_SIZE(mapinfo); i++) {
253 		struct mapinfo *mi = &mapinfo[i];
254 		resource_size_t length;
255 		resource_size_t addr;
256 
257 		if (!mi->rmap->valid)
258 			continue;
259 
260 		addr = phys_addr + mi->rmap->offset;
261 		length = mi->rmap->size;
262 		*(mi->addr) = devm_cxl_iomap_block(host, addr, length);
263 		if (!*(mi->addr))
264 			return -ENOMEM;
265 	}
266 
267 	return 0;
268 }
269 EXPORT_SYMBOL_NS_GPL(cxl_map_device_regs, CXL);
270 
cxl_decode_regblock(struct pci_dev * pdev,u32 reg_lo,u32 reg_hi,struct cxl_register_map * map)271 static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
272 				struct cxl_register_map *map)
273 {
274 	int bar = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BIR_MASK, reg_lo);
275 	u64 offset = ((u64)reg_hi << 32) |
276 		     (reg_lo & CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK);
277 
278 	if (offset > pci_resource_len(pdev, bar)) {
279 		dev_warn(&pdev->dev,
280 			 "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar,
281 			 &pdev->resource[bar], &offset, map->reg_type);
282 		return false;
283 	}
284 
285 	map->reg_type = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK, reg_lo);
286 	map->resource = pci_resource_start(pdev, bar) + offset;
287 	map->max_size = pci_resource_len(pdev, bar) - offset;
288 	return true;
289 }
290 
291 /**
292  * cxl_find_regblock_instance() - Locate a register block by type / index
293  * @pdev: The CXL PCI device to enumerate.
294  * @type: Register Block Indicator id
295  * @map: Enumeration output, clobbered on error
296  * @index: Index into which particular instance of a regblock wanted in the
297  *	   order found in register locator DVSEC.
298  *
299  * Return: 0 if register block enumerated, negative error code otherwise
300  *
301  * A CXL DVSEC may point to one or more register blocks, search for them
302  * by @type and @index.
303  */
cxl_find_regblock_instance(struct pci_dev * pdev,enum cxl_regloc_type type,struct cxl_register_map * map,int index)304 int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type,
305 			       struct cxl_register_map *map, int index)
306 {
307 	u32 regloc_size, regblocks;
308 	int instance = 0;
309 	int regloc, i;
310 
311 	*map = (struct cxl_register_map) {
312 		.host = &pdev->dev,
313 		.resource = CXL_RESOURCE_NONE,
314 	};
315 
316 	regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL,
317 					   CXL_DVSEC_REG_LOCATOR);
318 	if (!regloc)
319 		return -ENXIO;
320 
321 	pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);
322 	regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
323 
324 	regloc += CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET;
325 	regblocks = (regloc_size - CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET) / 8;
326 
327 	for (i = 0; i < regblocks; i++, regloc += 8) {
328 		u32 reg_lo, reg_hi;
329 
330 		pci_read_config_dword(pdev, regloc, &reg_lo);
331 		pci_read_config_dword(pdev, regloc + 4, &reg_hi);
332 
333 		if (!cxl_decode_regblock(pdev, reg_lo, reg_hi, map))
334 			continue;
335 
336 		if (map->reg_type == type) {
337 			if (index == instance)
338 				return 0;
339 			instance++;
340 		}
341 	}
342 
343 	map->resource = CXL_RESOURCE_NONE;
344 	return -ENODEV;
345 }
346 EXPORT_SYMBOL_NS_GPL(cxl_find_regblock_instance, CXL);
347 
348 /**
349  * cxl_find_regblock() - Locate register blocks by type
350  * @pdev: The CXL PCI device to enumerate.
351  * @type: Register Block Indicator id
352  * @map: Enumeration output, clobbered on error
353  *
354  * Return: 0 if register block enumerated, negative error code otherwise
355  *
356  * A CXL DVSEC may point to one or more register blocks, search for them
357  * by @type.
358  */
cxl_find_regblock(struct pci_dev * pdev,enum cxl_regloc_type type,struct cxl_register_map * map)359 int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type,
360 		      struct cxl_register_map *map)
361 {
362 	return cxl_find_regblock_instance(pdev, type, map, 0);
363 }
364 EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL);
365 
366 /**
367  * cxl_count_regblock() - Count instances of a given regblock type.
368  * @pdev: The CXL PCI device to enumerate.
369  * @type: Register Block Indicator id
370  *
371  * Some regblocks may be repeated. Count how many instances.
372  *
373  * Return: count of matching regblocks.
374  */
cxl_count_regblock(struct pci_dev * pdev,enum cxl_regloc_type type)375 int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type)
376 {
377 	struct cxl_register_map map;
378 	int rc, count = 0;
379 
380 	while (1) {
381 		rc = cxl_find_regblock_instance(pdev, type, &map, count);
382 		if (rc)
383 			return count;
384 		count++;
385 	}
386 }
387 EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, CXL);
388 
cxl_map_pmu_regs(struct pci_dev * pdev,struct cxl_pmu_regs * regs,struct cxl_register_map * map)389 int cxl_map_pmu_regs(struct pci_dev *pdev, struct cxl_pmu_regs *regs,
390 		     struct cxl_register_map *map)
391 {
392 	struct device *dev = &pdev->dev;
393 	resource_size_t phys_addr;
394 
395 	phys_addr = map->resource;
396 	regs->pmu = devm_cxl_iomap_block(dev, phys_addr, CXL_PMU_REGMAP_SIZE);
397 	if (!regs->pmu)
398 		return -ENOMEM;
399 
400 	return 0;
401 }
402 EXPORT_SYMBOL_NS_GPL(cxl_map_pmu_regs, CXL);
403 
cxl_map_regblock(struct cxl_register_map * map)404 static int cxl_map_regblock(struct cxl_register_map *map)
405 {
406 	struct device *host = map->host;
407 
408 	map->base = ioremap(map->resource, map->max_size);
409 	if (!map->base) {
410 		dev_err(host, "failed to map registers\n");
411 		return -ENOMEM;
412 	}
413 
414 	dev_dbg(host, "Mapped CXL Memory Device resource %pa\n", &map->resource);
415 	return 0;
416 }
417 
cxl_unmap_regblock(struct cxl_register_map * map)418 static void cxl_unmap_regblock(struct cxl_register_map *map)
419 {
420 	iounmap(map->base);
421 	map->base = NULL;
422 }
423 
cxl_probe_regs(struct cxl_register_map * map)424 static int cxl_probe_regs(struct cxl_register_map *map)
425 {
426 	struct cxl_component_reg_map *comp_map;
427 	struct cxl_device_reg_map *dev_map;
428 	struct device *host = map->host;
429 	void __iomem *base = map->base;
430 
431 	switch (map->reg_type) {
432 	case CXL_REGLOC_RBI_COMPONENT:
433 		comp_map = &map->component_map;
434 		cxl_probe_component_regs(host, base, comp_map);
435 		dev_dbg(host, "Set up component registers\n");
436 		break;
437 	case CXL_REGLOC_RBI_MEMDEV:
438 		dev_map = &map->device_map;
439 		cxl_probe_device_regs(host, base, dev_map);
440 		if (!dev_map->status.valid || !dev_map->mbox.valid ||
441 		    !dev_map->memdev.valid) {
442 			dev_err(host, "registers not found: %s%s%s\n",
443 				!dev_map->status.valid ? "status " : "",
444 				!dev_map->mbox.valid ? "mbox " : "",
445 				!dev_map->memdev.valid ? "memdev " : "");
446 			return -ENXIO;
447 		}
448 
449 		dev_dbg(host, "Probing device registers...\n");
450 		break;
451 	default:
452 		break;
453 	}
454 
455 	return 0;
456 }
457 
cxl_setup_regs(struct cxl_register_map * map)458 int cxl_setup_regs(struct cxl_register_map *map)
459 {
460 	int rc;
461 
462 	rc = cxl_map_regblock(map);
463 	if (rc)
464 		return rc;
465 
466 	rc = cxl_probe_regs(map);
467 	cxl_unmap_regblock(map);
468 
469 	return rc;
470 }
471 EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, CXL);
472 
__rcrb_to_component(struct device * dev,struct cxl_rcrb_info * ri,enum cxl_rcrb which)473 resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri,
474 				    enum cxl_rcrb which)
475 {
476 	resource_size_t component_reg_phys;
477 	resource_size_t rcrb = ri->base;
478 	void __iomem *addr;
479 	u32 bar0, bar1;
480 	u16 cmd;
481 	u32 id;
482 
483 	if (which == CXL_RCRB_UPSTREAM)
484 		rcrb += SZ_4K;
485 
486 	/*
487 	 * RCRB's BAR[0..1] point to component block containing CXL
488 	 * subsystem component registers. MEMBAR extraction follows
489 	 * the PCI Base spec here, esp. 64 bit extraction and memory
490 	 * ranges alignment (6.0, 7.5.1.2.1).
491 	 */
492 	if (!request_mem_region(rcrb, SZ_4K, "CXL RCRB"))
493 		return CXL_RESOURCE_NONE;
494 	addr = ioremap(rcrb, SZ_4K);
495 	if (!addr) {
496 		dev_err(dev, "Failed to map region %pr\n", addr);
497 		release_mem_region(rcrb, SZ_4K);
498 		return CXL_RESOURCE_NONE;
499 	}
500 
501 	id = readl(addr + PCI_VENDOR_ID);
502 	cmd = readw(addr + PCI_COMMAND);
503 	bar0 = readl(addr + PCI_BASE_ADDRESS_0);
504 	bar1 = readl(addr + PCI_BASE_ADDRESS_1);
505 	iounmap(addr);
506 	release_mem_region(rcrb, SZ_4K);
507 
508 	/*
509 	 * Sanity check, see CXL 3.0 Figure 9-8 CXL Device that Does Not
510 	 * Remap Upstream Port and Component Registers
511 	 */
512 	if (id == U32_MAX) {
513 		if (which == CXL_RCRB_DOWNSTREAM)
514 			dev_err(dev, "Failed to access Downstream Port RCRB\n");
515 		return CXL_RESOURCE_NONE;
516 	}
517 	if (!(cmd & PCI_COMMAND_MEMORY))
518 		return CXL_RESOURCE_NONE;
519 	/* The RCRB is a Memory Window, and the MEM_TYPE_1M bit is obsolete */
520 	if (bar0 & (PCI_BASE_ADDRESS_MEM_TYPE_1M | PCI_BASE_ADDRESS_SPACE_IO))
521 		return CXL_RESOURCE_NONE;
522 
523 	component_reg_phys = bar0 & PCI_BASE_ADDRESS_MEM_MASK;
524 	if (bar0 & PCI_BASE_ADDRESS_MEM_TYPE_64)
525 		component_reg_phys |= ((u64)bar1) << 32;
526 
527 	if (!component_reg_phys)
528 		return CXL_RESOURCE_NONE;
529 
530 	/* MEMBAR is block size (64k) aligned. */
531 	if (!IS_ALIGNED(component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE))
532 		return CXL_RESOURCE_NONE;
533 
534 	return component_reg_phys;
535 }
536 
cxl_rcd_component_reg_phys(struct device * dev,struct cxl_dport * dport)537 resource_size_t cxl_rcd_component_reg_phys(struct device *dev,
538 					   struct cxl_dport *dport)
539 {
540 	if (!dport->rch)
541 		return CXL_RESOURCE_NONE;
542 	return __rcrb_to_component(dev, &dport->rcrb, CXL_RCRB_UPSTREAM);
543 }
544 EXPORT_SYMBOL_NS_GPL(cxl_rcd_component_reg_phys, CXL);
545