89 #define AFUD_READ(afu, off)		in_be64(afu->native->afu_desc_mmio + off)  argument
90 #define AFUD_READ_LE(afu, off)		in_le64(afu->native->afu_desc_mmio + off)  argument
94 #define AFUD_READ_INFO(afu)		AFUD_READ(afu, 0x0)  argument
103 #define AFUD_READ_CR(afu)		AFUD_READ(afu, 0x20)  argument
105 #define AFUD_READ_CR_OFF(afu)		AFUD_READ(afu, 0x28)  argument
106 #define AFUD_READ_PPPSA(afu)		AFUD_READ(afu, 0x30)  argument
110 #define AFUD_READ_PPPSA_OFF(afu)	AFUD_READ(afu, 0x38)  argument
111 #define AFUD_READ_EB(afu)		AFUD_READ(afu, 0x40)  argument
113 #define AFUD_READ_EB_OFF(afu)		AFUD_READ(afu, 0x48)  argument
268 static void dump_afu_descriptor(struct cxl_afu *afu)  in dump_afu_descriptor()  argument
274 	dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)  in dump_afu_descriptor()
276 	val = AFUD_READ_INFO(afu);  in dump_afu_descriptor()
283 	val = AFUD_READ(afu, 0x8);  in dump_afu_descriptor()
285 	val = AFUD_READ(afu, 0x10);  in dump_afu_descriptor()
287 	val = AFUD_READ(afu, 0x18);  in dump_afu_descriptor()
290 	val = AFUD_READ_CR(afu);  in dump_afu_descriptor()
295 	val = AFUD_READ_CR_OFF(afu);  in dump_afu_descriptor()
299 	val = AFUD_READ_PPPSA(afu);  in dump_afu_descriptor()
303 	val = AFUD_READ_PPPSA_OFF(afu);  in dump_afu_descriptor()
306 	val = AFUD_READ_EB(afu);  in dump_afu_descriptor()
310 	val = AFUD_READ_EB_OFF(afu);  in dump_afu_descriptor()
314 		val = AFUD_READ_LE(afu, afu_cr_off + i * afu_cr_len);  in dump_afu_descriptor()
637 static int init_implementation_afu_regs_psl9(struct cxl_afu *afu)  in init_implementation_afu_regs_psl9()  argument
642 static int init_implementation_afu_regs_psl8(struct cxl_afu *afu)  in init_implementation_afu_regs_psl8()  argument
645 	cxl_p1n_write(afu, CXL_PSL_APCALLOC_A, 0xFFFFFFFEFEFEFEFEULL);  in init_implementation_afu_regs_psl8()
647 	cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL);  in init_implementation_afu_regs_psl8()
649 	cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL);  in init_implementation_afu_regs_psl8()
650 	cxl_p1n_write(afu, CXL_PSL_RXCTL_A, CXL_PSL_RXCTL_AFUHP_4S);  in init_implementation_afu_regs_psl8()
782 static int pci_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in pci_map_slice_regs()  argument
788 	p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);  in pci_map_slice_regs()
789 	p2n_base = p2_base(dev) + (afu->slice * p2n_size);  in pci_map_slice_regs()
790 	afu->psn_phys = p2_base(dev) + (adapter->native->ps_off + (afu->slice * adapter->ps_size));  in pci_map_slice_regs()
791 …afu_desc = p2_base(dev) + adapter->native->afu_desc_off + (afu->slice * adapter->native->afu_desc_…  in pci_map_slice_regs()
793 	if (!(afu->native->p1n_mmio = ioremap(p1n_base, p1n_size)))  in pci_map_slice_regs()
795 	if (!(afu->p2n_mmio = ioremap(p2n_base, p2n_size)))  in pci_map_slice_regs()
798 		if (!(afu->native->afu_desc_mmio = ioremap(afu_desc, adapter->native->afu_desc_size)))  in pci_map_slice_regs()
804 	iounmap(afu->p2n_mmio);  in pci_map_slice_regs()
806 	iounmap(afu->native->p1n_mmio);  in pci_map_slice_regs()
808 	dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");  in pci_map_slice_regs()
812 static void pci_unmap_slice_regs(struct cxl_afu *afu)  in pci_unmap_slice_regs()  argument
814 	if (afu->p2n_mmio) {  in pci_unmap_slice_regs()
815 		iounmap(afu->p2n_mmio);  in pci_unmap_slice_regs()
816 		afu->p2n_mmio = NULL;  in pci_unmap_slice_regs()
818 	if (afu->native->p1n_mmio) {  in pci_unmap_slice_regs()
819 		iounmap(afu->native->p1n_mmio);  in pci_unmap_slice_regs()
820 		afu->native->p1n_mmio = NULL;  in pci_unmap_slice_regs()
822 	if (afu->native->afu_desc_mmio) {  in pci_unmap_slice_regs()
823 		iounmap(afu->native->afu_desc_mmio);  in pci_unmap_slice_regs()
824 		afu->native->afu_desc_mmio = NULL;  in pci_unmap_slice_regs()
830 	struct cxl_afu *afu = to_cxl_afu(dev);  in cxl_pci_release_afu()  local
834 	idr_destroy(&afu->contexts_idr);  in cxl_pci_release_afu()
835 	cxl_release_spa(afu);  in cxl_pci_release_afu()
837 	kfree(afu->native);  in cxl_pci_release_afu()
838 	kfree(afu);  in cxl_pci_release_afu()
842 static int cxl_read_afu_descriptor(struct cxl_afu *afu)  in cxl_read_afu_descriptor()  argument
846 	val = AFUD_READ_INFO(afu);  in cxl_read_afu_descriptor()
847 	afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val);  in cxl_read_afu_descriptor()
848 	afu->max_procs_virtualised = AFUD_NUM_PROCS(val);  in cxl_read_afu_descriptor()
849 	afu->crs_num = AFUD_NUM_CRS(val);  in cxl_read_afu_descriptor()
852 		afu->modes_supported |= CXL_MODE_DIRECTED;  in cxl_read_afu_descriptor()
854 		afu->modes_supported |= CXL_MODE_DEDICATED;  in cxl_read_afu_descriptor()
856 		afu->modes_supported |= CXL_MODE_TIME_SLICED;  in cxl_read_afu_descriptor()
858 	val = AFUD_READ_PPPSA(afu);  in cxl_read_afu_descriptor()
859 	afu->pp_size = AFUD_PPPSA_LEN(val) * 4096;  in cxl_read_afu_descriptor()
860 	afu->psa = AFUD_PPPSA_PSA(val);  in cxl_read_afu_descriptor()
861 	if ((afu->pp_psa = AFUD_PPPSA_PP(val)))  in cxl_read_afu_descriptor()
862 		afu->native->pp_offset = AFUD_READ_PPPSA_OFF(afu);  in cxl_read_afu_descriptor()
864 	val = AFUD_READ_CR(afu);  in cxl_read_afu_descriptor()
865 	afu->crs_len = AFUD_CR_LEN(val) * 256;  in cxl_read_afu_descriptor()
866 	afu->crs_offset = AFUD_READ_CR_OFF(afu);  in cxl_read_afu_descriptor()
870 	afu->eb_len = AFUD_EB_LEN(AFUD_READ_EB(afu)) * 4096;  in cxl_read_afu_descriptor()
871 	afu->eb_offset = AFUD_READ_EB_OFF(afu);  in cxl_read_afu_descriptor()
874 	if (EXTRACT_PPC_BITS(afu->eb_offset, 0, 11) != 0) {  in cxl_read_afu_descriptor()
875 		dev_warn(&afu->dev,  in cxl_read_afu_descriptor()
877 			 afu->eb_offset);  in cxl_read_afu_descriptor()
878 		dev_info(&afu->dev,  in cxl_read_afu_descriptor()
881 		afu->eb_len = 0;  in cxl_read_afu_descriptor()
887 static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu)  in cxl_afu_descriptor_looks_ok()  argument
892 	if (afu->psa && afu->adapter->ps_size <  in cxl_afu_descriptor_looks_ok()
893 			(afu->native->pp_offset + afu->pp_size*afu->max_procs_virtualised)) {  in cxl_afu_descriptor_looks_ok()
894 		dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");  in cxl_afu_descriptor_looks_ok()
898 	if (afu->pp_psa && (afu->pp_size < PAGE_SIZE))  in cxl_afu_descriptor_looks_ok()
899 		dev_warn(&afu->dev, "AFU uses pp_size(%#016llx) < PAGE_SIZE per-process PSA!\n", afu->pp_size);  in cxl_afu_descriptor_looks_ok()
901 	for (i = 0; i < afu->crs_num; i++) {  in cxl_afu_descriptor_looks_ok()
902 		rc = cxl_ops->afu_cr_read32(afu, i, 0, &val);  in cxl_afu_descriptor_looks_ok()
904 			dev_err(&afu->dev, "ABORTING: AFU configuration record %i is invalid\n", i);  in cxl_afu_descriptor_looks_ok()
909 	if ((afu->modes_supported & ~CXL_MODE_DEDICATED) && afu->max_procs_virtualised == 0) {  in cxl_afu_descriptor_looks_ok()
920 		dev_err(&afu->dev, "AFU does not support any processes\n");  in cxl_afu_descriptor_looks_ok()
927 static int sanitise_afu_regs_psl9(struct cxl_afu *afu)  in sanitise_afu_regs_psl9()  argument
936 	reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);  in sanitise_afu_regs_psl9()
938 		dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#016llx\n", reg);  in sanitise_afu_regs_psl9()
939 		if (cxl_ops->afu_reset(afu))  in sanitise_afu_regs_psl9()
941 		if (cxl_afu_disable(afu))  in sanitise_afu_regs_psl9()
943 		if (cxl_psl_purge(afu))  in sanitise_afu_regs_psl9()
946 	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);  in sanitise_afu_regs_psl9()
947 	cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);  in sanitise_afu_regs_psl9()
948 	reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);  in sanitise_afu_regs_psl9()
950 		dev_warn(&afu->dev, "AFU had pending DSISR: %#016llx\n", reg);  in sanitise_afu_regs_psl9()
952 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);  in sanitise_afu_regs_psl9()
954 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);  in sanitise_afu_regs_psl9()
956 	if (afu->adapter->native->sl_ops->register_serr_irq) {  in sanitise_afu_regs_psl9()
957 		reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);  in sanitise_afu_regs_psl9()
960 				dev_warn(&afu->dev, "AFU had pending SERR: %#016llx\n", reg);  in sanitise_afu_regs_psl9()
961 			cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);  in sanitise_afu_regs_psl9()
964 	reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);  in sanitise_afu_regs_psl9()
966 		dev_warn(&afu->dev, "AFU had pending error status: %#016llx\n", reg);  in sanitise_afu_regs_psl9()
967 		cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);  in sanitise_afu_regs_psl9()
973 static int sanitise_afu_regs_psl8(struct cxl_afu *afu)  in sanitise_afu_regs_psl8()  argument
982 	reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);  in sanitise_afu_regs_psl8()
984 		dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#016llx\n", reg);  in sanitise_afu_regs_psl8()
985 		if (cxl_ops->afu_reset(afu))  in sanitise_afu_regs_psl8()
987 		if (cxl_afu_disable(afu))  in sanitise_afu_regs_psl8()
989 		if (cxl_psl_purge(afu))  in sanitise_afu_regs_psl8()
992 	cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
993 	cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
994 	cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
995 	cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
996 	cxl_p1n_write(afu, CXL_PSL_SPOffset_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
997 	cxl_p1n_write(afu, CXL_HAURP_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
998 	cxl_p2n_write(afu, CXL_CSRP_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
999 	cxl_p2n_write(afu, CXL_AURP1_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
1000 	cxl_p2n_write(afu, CXL_AURP0_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
1001 	cxl_p2n_write(afu, CXL_SSTP1_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
1002 	cxl_p2n_write(afu, CXL_SSTP0_An, 0x0000000000000000);  in sanitise_afu_regs_psl8()
1003 	reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);  in sanitise_afu_regs_psl8()
1005 		dev_warn(&afu->dev, "AFU had pending DSISR: %#016llx\n", reg);  in sanitise_afu_regs_psl8()
1007 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);  in sanitise_afu_regs_psl8()
1009 			cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);  in sanitise_afu_regs_psl8()
1011 	if (afu->adapter->native->sl_ops->register_serr_irq) {  in sanitise_afu_regs_psl8()
1012 		reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);  in sanitise_afu_regs_psl8()
1015 				dev_warn(&afu->dev, "AFU had pending SERR: %#016llx\n", reg);  in sanitise_afu_regs_psl8()
1016 			cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);  in sanitise_afu_regs_psl8()
1019 	reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);  in sanitise_afu_regs_psl8()
1021 		dev_warn(&afu->dev, "AFU had pending error status: %#016llx\n", reg);  in sanitise_afu_regs_psl8()
1022 		cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);  in sanitise_afu_regs_psl8()
1035 ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf,  in cxl_pci_afu_read_err_buffer()  argument
1041 	const void __iomem *ebuf = afu->native->afu_desc_mmio + afu->eb_offset;  in cxl_pci_afu_read_err_buffer()
1043 	if (count == 0 || off < 0 || (size_t)off >= afu->eb_len)  in cxl_pci_afu_read_err_buffer()
1047 	count = min((size_t)(afu->eb_len - off), count);  in cxl_pci_afu_read_err_buffer()
1072 static int pci_configure_afu(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)  in pci_configure_afu()  argument
1076 	if ((rc = pci_map_slice_regs(afu, adapter, dev)))  in pci_configure_afu()
1080 		rc = adapter->native->sl_ops->sanitise_afu_regs(afu);  in pci_configure_afu()
1086 	if ((rc = cxl_ops->afu_reset(afu)))  in pci_configure_afu()
1090 		dump_afu_descriptor(afu);  in pci_configure_afu()
1092 	if ((rc = cxl_read_afu_descriptor(afu)))  in pci_configure_afu()
1095 	if ((rc = cxl_afu_descriptor_looks_ok(afu)))  in pci_configure_afu()
1099 		if ((rc = adapter->native->sl_ops->afu_regs_init(afu)))  in pci_configure_afu()
1103 		if ((rc = adapter->native->sl_ops->register_serr_irq(afu)))  in pci_configure_afu()
1106 	if ((rc = cxl_native_register_psl_irq(afu)))  in pci_configure_afu()
1109 	atomic_set(&afu->configured_state, 0);  in pci_configure_afu()
1114 		adapter->native->sl_ops->release_serr_irq(afu);  in pci_configure_afu()
1116 	pci_unmap_slice_regs(afu);  in pci_configure_afu()
1120 static void pci_deconfigure_afu(struct cxl_afu *afu)  in pci_deconfigure_afu()  argument
1126 	if (atomic_read(&afu->configured_state) != -1) {  in pci_deconfigure_afu()
1127 		while (atomic_cmpxchg(&afu->configured_state, 0, -1) != -1)  in pci_deconfigure_afu()
1130 	cxl_native_release_psl_irq(afu);  in pci_deconfigure_afu()
1131 	if (afu->adapter->native->sl_ops->release_serr_irq)  in pci_deconfigure_afu()
1132 		afu->adapter->native->sl_ops->release_serr_irq(afu);  in pci_deconfigure_afu()
1133 	pci_unmap_slice_regs(afu);  in pci_deconfigure_afu()
1138 	struct cxl_afu *afu;  in pci_init_afu()  local
1141 	afu = cxl_alloc_afu(adapter, slice);  in pci_init_afu()
1142 	if (!afu)  in pci_init_afu()
1145 	afu->native = kzalloc(sizeof(struct cxl_afu_native), GFP_KERNEL);  in pci_init_afu()
1146 	if (!afu->native)  in pci_init_afu()
1149 	mutex_init(&afu->native->spa_mutex);  in pci_init_afu()
1151 	rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice);  in pci_init_afu()
1155 	rc = pci_configure_afu(afu, adapter, dev);  in pci_init_afu()
1160 	cxl_debugfs_afu_add(afu);  in pci_init_afu()
1166 	if ((rc = cxl_register_afu(afu)))  in pci_init_afu()
1169 	if ((rc = cxl_sysfs_afu_add(afu)))  in pci_init_afu()
1172 	adapter->afu[afu->slice] = afu;  in pci_init_afu()
1174 	if ((rc = cxl_pci_vphb_add(afu)))  in pci_init_afu()
1175 		dev_info(&afu->dev, "Can't register vPHB\n");  in pci_init_afu()
1180 	pci_deconfigure_afu(afu);  in pci_init_afu()
1181 	cxl_debugfs_afu_remove(afu);  in pci_init_afu()
1182 	device_unregister(&afu->dev);  in pci_init_afu()
1186 	kfree(afu->native);  in pci_init_afu()
1188 	kfree(afu);  in pci_init_afu()
1193 static void cxl_pci_remove_afu(struct cxl_afu *afu)  in cxl_pci_remove_afu()  argument
1197 	if (!afu)  in cxl_pci_remove_afu()
1200 	cxl_pci_vphb_remove(afu);  in cxl_pci_remove_afu()
1201 	cxl_sysfs_afu_remove(afu);  in cxl_pci_remove_afu()
1202 	cxl_debugfs_afu_remove(afu);  in cxl_pci_remove_afu()
1204 	spin_lock(&afu->adapter->afu_list_lock);  in cxl_pci_remove_afu()
1205 	afu->adapter->afu[afu->slice] = NULL;  in cxl_pci_remove_afu()
1206 	spin_unlock(&afu->adapter->afu_list_lock);  in cxl_pci_remove_afu()
1208 	cxl_context_detach_all(afu);  in cxl_pci_remove_afu()
1209 	cxl_ops->afu_deactivate_mode(afu, afu->current_mode);  in cxl_pci_remove_afu()
1211 	pci_deconfigure_afu(afu);  in cxl_pci_remove_afu()
1212 	device_unregister(&afu->dev);  in cxl_pci_remove_afu()
1567 		if (adapter->afu[slice])  in cxl_stop_trace_psl8()
1568 			cxl_p1n_write(adapter->afu[slice], CXL_PSL_SLICE_TRACE,  in cxl_stop_trace_psl8()
1769 		rc = cxl_afu_select_best_mode(adapter->afu[slice]);  in cxl_probe()
1780 	struct cxl_afu *afu;  in cxl_remove()  local
1788 		afu = adapter->afu[i];  in cxl_remove()
1789 		cxl_pci_remove_afu(afu);  in cxl_remove()
1794 static pci_ers_result_t cxl_vphb_error_detected(struct cxl_afu *afu,  in cxl_vphb_error_detected()  argument
1806 	if (afu == NULL || afu->phb == NULL)  in cxl_vphb_error_detected()
1809 	list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_vphb_error_detected()
1834 	struct cxl_afu *afu;  in cxl_pci_error_detected()  local
1849 			afu = adapter->afu[i];  in cxl_pci_error_detected()
1854 			cxl_vphb_error_detected(afu, state);  in cxl_pci_error_detected()
1943 		afu = adapter->afu[i];  in cxl_pci_error_detected()
1945 		if (afu == NULL)  in cxl_pci_error_detected()
1948 		afu_result = cxl_vphb_error_detected(afu, state);  in cxl_pci_error_detected()
1949 		cxl_context_detach_all(afu);  in cxl_pci_error_detected()
1950 		cxl_ops->afu_deactivate_mode(afu, afu->current_mode);  in cxl_pci_error_detected()
1951 		pci_deconfigure_afu(afu);  in cxl_pci_error_detected()
1976 	struct cxl_afu *afu;  in cxl_pci_slot_reset()  local
1997 		afu = adapter->afu[i];  in cxl_pci_slot_reset()
1999 		if (afu == NULL)  in cxl_pci_slot_reset()
2002 		if (pci_configure_afu(afu, adapter, pdev))  in cxl_pci_slot_reset()
2005 		if (cxl_afu_select_best_mode(afu))  in cxl_pci_slot_reset()
2008 		if (afu->phb == NULL)  in cxl_pci_slot_reset()
2011 		list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_pci_slot_reset()
2026 			if (cxl_ops->afu_check_and_enable(afu))  in cxl_pci_slot_reset()
2068 	struct cxl_afu *afu;  in cxl_pci_resume()  local
2080 		afu = adapter->afu[i];  in cxl_pci_resume()
2082 		if (afu == NULL || afu->phb == NULL)  in cxl_pci_resume()
2085 		list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {  in cxl_pci_resume()