228 	afu->native->spa_order = -1;  in cxl_alloc_spa()
230 		afu->native->spa_order++;  in cxl_alloc_spa()
231 		spa_size = (1 << afu->native->spa_order) * PAGE_SIZE;  in cxl_alloc_spa()
235 					afu->native->spa_max_procs, afu->native->spa_size);  in cxl_alloc_spa()
237 				afu->num_procs = afu->native->spa_max_procs;  in cxl_alloc_spa()
241 		afu->native->spa_size = spa_size;  in cxl_alloc_spa()
242 		afu->native->spa_max_procs = spa_max_procs(afu->native->spa_size);  in cxl_alloc_spa()
243 	} while (afu->native->spa_max_procs < afu->num_procs);  in cxl_alloc_spa()
245 	if (!(afu->native->spa = (struct cxl_process_element *)  in cxl_alloc_spa()
246 	      __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->native->spa_order))) {  in cxl_alloc_spa()
251 		 1<<afu->native->spa_order, afu->native->spa_max_procs, afu->num_procs);  in cxl_alloc_spa()
260 	afu->native->sw_command_status = (__be64 *)((char *)afu->native->spa +  in attach_spa()
261 					    ((afu->native->spa_max_procs + 3) * 128));  in attach_spa()
263 	spap = virt_to_phys(afu->native->spa) & CXL_PSL_SPAP_Addr;  in attach_spa()
264 	spap |= ((afu->native->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;  in attach_spa()
267 		afu->native->spa, afu->native->spa_max_procs,  in attach_spa()
268 		afu->native->sw_command_status, spap);  in attach_spa()
279 	if (afu->native->spa) {  in cxl_release_spa()
280 		free_pages((unsigned long) afu->native->spa,  in cxl_release_spa()
281 			afu->native->spa_order);  in cxl_release_spa()
282 		afu->native->spa = NULL;  in cxl_release_spa()
358 	if (adapter->native->no_data_cache) {  in cxl_data_cache_flush()
414 	WARN_ON(!mutex_is_locked(&ctx->afu->native->spa_mutex));  in slb_invalid()
444 	*(ctx->afu->native->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);  in do_process_element_cmd()
458 		state = be64_to_cpup(ctx->afu->native->sw_command_status);  in do_process_element_cmd()
486 	mutex_lock(&ctx->afu->native->spa_mutex);  in add_process_element()
491 	mutex_unlock(&ctx->afu->native->spa_mutex);  in add_process_element()
503 	mutex_lock(&ctx->afu->native->spa_mutex);  in terminate_process_element()
514 	mutex_unlock(&ctx->afu->native->spa_mutex);  in terminate_process_element()
522 	mutex_lock(&ctx->afu->native->spa_mutex);  in remove_process_element()
536 	mutex_unlock(&ctx->afu->native->spa_mutex);  in remove_process_element()
548 			(ctx->afu->native->pp_offset + ctx->afu->pp_size * ctx->pe);  in cxl_assign_psn_space()
560 	if (afu->native->spa == NULL) {  in activate_afu_directed()
708 		ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;  in process_element_entry_psl9()
771 		ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;  in cxl_attach_afu_directed_psl8()
822 	if (afu->adapter->native->sl_ops->needs_reset_before_disable)  in deactivate_afu_directed()
842 	if (afu->native->spa == NULL) {  in cxl_activate_dedicated_process_psl9()
915 	if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)  in cxl_attach_dedicated_process_psl9()
916 		afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);  in cxl_attach_dedicated_process_psl9()
951 	if (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes)  in cxl_attach_dedicated_process_psl8()
952 		afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);  in cxl_attach_dedicated_process_psl8()
1003 	    (afu->adapter->native->sl_ops->activate_dedicated_process))  in native_afu_activate_mode()
1004 		return afu->adapter->native->sl_ops->activate_dedicated_process(afu);  in native_afu_activate_mode()
1019 	    (ctx->afu->adapter->native->sl_ops->attach_afu_directed))  in native_attach_process()
1020 		return ctx->afu->adapter->native->sl_ops->attach_afu_directed(ctx, wed, amr);  in native_attach_process()
1023 	    (ctx->afu->adapter->native->sl_ops->attach_dedicated_process))  in native_attach_process()
1024 		return ctx->afu->adapter->native->sl_ops->attach_dedicated_process(ctx, wed, amr);  in native_attach_process()
1058 	    (ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes))  in native_update_ivtes()
1059 		return ctx->afu->adapter->native->sl_ops->update_dedicated_ivtes(ctx);  in native_update_ivtes()
1111 	if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {  in cxl_native_irq_dump_regs_psl9()
1128 	if (ctx->afu->adapter->native->sl_ops->register_serr_irq) {  in cxl_native_irq_dump_regs_psl8()
1142 	if (ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers)  in native_handle_psl_slice_error()
1143 		ctx->afu->adapter->native->sl_ops->psl_irq_dump_registers(ctx);  in native_handle_psl_slice_error()
1145 	if (ctx->afu->adapter->native->sl_ops->debugfs_stop_trace) {  in native_handle_psl_slice_error()
1147 		ctx->afu->adapter->native->sl_ops->debugfs_stop_trace(ctx->afu->adapter);  in native_handle_psl_slice_error()
1193 		if (afu->adapter->native->sl_ops->fail_irq)  in native_irq_multiplexed()
1194 			return afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);  in native_irq_multiplexed()
1201 		if (afu->adapter->native->sl_ops->handle_interrupt)  in native_irq_multiplexed()
1202 			ret = afu->adapter->native->sl_ops->handle_interrupt(irq, ctx, &irq_info);  in native_irq_multiplexed()
1212 	if (afu->adapter->native->sl_ops->fail_irq)  in native_irq_multiplexed()
1213 		ret = afu->adapter->native->sl_ops->fail_irq(afu, &irq_info);  in native_irq_multiplexed()
1313 	if (adapter->native->sl_ops->debugfs_stop_trace) {  in native_irq_err()
1315 		adapter->native->sl_ops->debugfs_stop_trace(adapter);  in native_irq_err()
1318 	if (adapter->native->sl_ops->err_irq_dump_registers)  in native_irq_err()
1319 		adapter->native->sl_ops->err_irq_dump_registers(adapter);  in native_irq_err()
1334 				       &adapter->native->err_hwirq,  in cxl_native_register_psl_err_irq()
1335 				       &adapter->native->err_virq,  in cxl_native_register_psl_err_irq()
1342 	cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->native->err_hwirq & 0xffff);  in cxl_native_register_psl_err_irq()
1349 	if (adapter->native->err_virq == 0 ||  in cxl_native_release_psl_err_irq()
1350 	    adapter->native->err_virq !=  in cxl_native_release_psl_err_irq()
1351 	    irq_find_mapping(NULL, adapter->native->err_hwirq))  in cxl_native_release_psl_err_irq()
1355 	cxl_unmap_irq(adapter->native->err_virq, adapter);  in cxl_native_release_psl_err_irq()
1356 	cxl_ops->release_one_irq(adapter, adapter->native->err_hwirq);  in cxl_native_release_psl_err_irq()
1358 	adapter->native->err_virq = 0;  in cxl_native_release_psl_err_irq()
1417 				    afu, &afu->native->psl_hwirq, &afu->native->psl_virq,  in cxl_native_register_psl_irq()
1427 	if (afu->native->psl_virq == 0 ||  in cxl_native_release_psl_irq()
1428 	    afu->native->psl_virq !=  in cxl_native_release_psl_irq()
1429 	    irq_find_mapping(NULL, afu->native->psl_hwirq))  in cxl_native_release_psl_irq()
1432 	cxl_unmap_irq(afu->native->psl_virq, afu);  in cxl_native_release_psl_irq()
1433 	cxl_ops->release_one_irq(afu->adapter, afu->native->psl_hwirq);  in cxl_native_release_psl_irq()
1435 	afu->native->psl_virq = 0;  in cxl_native_release_psl_irq()
1480 	*out = in_le64(afu->native->afu_desc_mmio + afu->crs_offset +  in native_afu_cr_read64()
1491 	*out = in_le32(afu->native->afu_desc_mmio + afu->crs_offset +  in native_afu_cr_read32()
1526 	out_le32(afu->native->afu_desc_mmio + afu->crs_offset +  in native_afu_cr_write32()