26 static int vgic_its_save_tables_v0(struct vgic_its *its);
27 static int vgic_its_restore_tables_v0(struct vgic_its *its);
28 static int vgic_its_commit_v0(struct vgic_its *its);
171 	int (*save_tables)(struct vgic_its *its);
172 	int (*restore_tables)(struct vgic_its *its);
173 	int (*commit)(struct vgic_its *its);
192 inline const struct vgic_its_abi *vgic_its_get_abi(struct vgic_its *its)  in vgic_its_get_abi()  argument
194 	return &its_table_abi_versions[its->abi_rev];  in vgic_its_get_abi()
197 static int vgic_its_set_abi(struct vgic_its *its, u32 rev)  in vgic_its_set_abi()  argument
201 	its->abi_rev = rev;  in vgic_its_set_abi()
202 	abi = vgic_its_get_abi(its);  in vgic_its_set_abi()
203 	return abi->commit(its);  in vgic_its_set_abi()
210 static struct its_device *find_its_device(struct vgic_its *its, u32 device_id)  in find_its_device()  argument
214 	list_for_each_entry(device, &its->device_list, dev_list)  in find_its_device()
226 static struct its_ite *find_ite(struct vgic_its *its, u32 device_id,  in find_ite()  argument
232 	device = find_its_device(its, device_id);  in find_ite()
244 #define for_each_lpi_its(dev, ite, its) \  argument
245 	list_for_each_entry(dev, &(its)->device_list, dev_list) \
259 static struct its_collection *find_collection(struct vgic_its *its, int coll_id)  in find_collection()  argument
263 	list_for_each_entry(collection, &its->collection_list, coll_list) {  in find_collection()
402 static void update_affinity_collection(struct kvm *kvm, struct vgic_its *its,  in update_affinity_collection()  argument
408 	for_each_lpi_its(device, ite, its) {  in update_affinity_collection()
480 					      struct vgic_its *its,  in vgic_mmio_read_its_typer()  argument
483 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_mmio_read_its_typer()
502 					     struct vgic_its *its,  in vgic_mmio_read_its_iidr()  argument
507 	val = (its->abi_rev << GITS_IIDR_REV_SHIFT) & GITS_IIDR_REV_MASK;  in vgic_mmio_read_its_iidr()
513 					    struct vgic_its *its,  in vgic_mmio_uaccess_write_its_iidr()  argument
521 	return vgic_its_set_abi(its, rev);  in vgic_mmio_uaccess_write_its_iidr()
525 					       struct vgic_its *its,  in vgic_mmio_read_its_idregs()  argument
600 static void vgic_its_cache_translation(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cache_translation()  argument
623 	db = its->vgic_its_base + GITS_TRANSLATER;  in vgic_its_cache_translation()
676 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_resolve_lpi()  argument
682 	if (!its->enabled)  in vgic_its_resolve_lpi()
685 	ite = find_ite(its, devid, eventid);  in vgic_its_resolve_lpi()
696 	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);  in vgic_its_resolve_lpi()
727 	return iodev->its;  in vgic_msi_to_its()
737 static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_trigger_msi()  argument
744 	err = vgic_its_resolve_lpi(kvm, its, devid, eventid, &irq);  in vgic_its_trigger_msi()
786 	struct vgic_its *its;  in vgic_its_inject_msi()  local
792 	its = vgic_msi_to_its(kvm, msi);  in vgic_its_inject_msi()
793 	if (IS_ERR(its))  in vgic_its_inject_msi()
794 		return PTR_ERR(its);  in vgic_its_inject_msi()
796 	mutex_lock(&its->its_lock);  in vgic_its_inject_msi()
797 	ret = vgic_its_trigger_msi(kvm, its, msi->devid, msi->data);  in vgic_its_inject_msi()
798 	mutex_unlock(&its->its_lock);  in vgic_its_inject_msi()
849 static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_discard()  argument
856 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_discard()
876 static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_movi()  argument
886 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_movi()
893 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_movi()
905 static bool __is_visible_gfn_locked(struct vgic_its *its, gpa_t gpa)  in __is_visible_gfn_locked()  argument
911 	idx = srcu_read_lock(&its->dev->kvm->srcu);  in __is_visible_gfn_locked()
912 	ret = kvm_is_visible_gfn(its->dev->kvm, gfn);  in __is_visible_gfn_locked()
913 	srcu_read_unlock(&its->dev->kvm->srcu, idx);  in __is_visible_gfn_locked()
924 static bool vgic_its_check_id(struct vgic_its *its, u64 baser, u32 id,  in vgic_its_check_id()  argument
958 		return __is_visible_gfn_locked(its, addr);  in vgic_its_check_id()
967 	if (kvm_read_guest_lock(its->dev->kvm,  in vgic_its_check_id()
988 	return __is_visible_gfn_locked(its, indirect_ptr);  in vgic_its_check_id()
995 static bool vgic_its_check_event_id(struct vgic_its *its, struct its_device *device,  in vgic_its_check_event_id()  argument
998 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_check_event_id()
1007 	return __is_visible_gfn_locked(its, gpa);  in vgic_its_check_event_id()
1014 static int vgic_its_alloc_collection(struct vgic_its *its,  in vgic_its_alloc_collection()  argument
1027 	list_add_tail(&collection->coll_list, &its->collection_list);  in vgic_its_alloc_collection()
1033 static void vgic_its_free_collection(struct vgic_its *its, u32 coll_id)  in vgic_its_free_collection()  argument
1044 	collection = find_collection(its, coll_id);  in vgic_its_free_collection()
1048 	for_each_lpi_its(device, ite, its)  in vgic_its_free_collection()
1079 static int vgic_its_cmd_handle_mapi(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapi()  argument
1092 	device = find_its_device(its, device_id);  in vgic_its_cmd_handle_mapi()
1096 	if (!vgic_its_check_event_id(its, device, event_id))  in vgic_its_cmd_handle_mapi()
1108 	if (find_ite(its, device_id, event_id))  in vgic_its_cmd_handle_mapi()
1111 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1115 		if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))  in vgic_its_cmd_handle_mapi()
1118 		ret = vgic_its_alloc_collection(its, &collection, coll_id);  in vgic_its_cmd_handle_mapi()
1127 			vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1137 			vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapi()
1166 static void vgic_its_free_device_list(struct kvm *kvm, struct vgic_its *its)  in vgic_its_free_device_list()  argument
1170 	list_for_each_entry_safe(cur, temp, &its->device_list, dev_list)  in vgic_its_free_device_list()
1175 static void vgic_its_free_collection_list(struct kvm *kvm, struct vgic_its *its)  in vgic_its_free_collection_list()  argument
1179 	list_for_each_entry_safe(cur, temp, &its->collection_list, coll_list)  in vgic_its_free_collection_list()
1180 		vgic_its_free_collection(its, cur->collection_id);  in vgic_its_free_collection_list()
1184 static struct its_device *vgic_its_alloc_device(struct vgic_its *its,  in vgic_its_alloc_device()  argument
1199 	list_add_tail(&device->dev_list, &its->device_list);  in vgic_its_alloc_device()
1207 static int vgic_its_cmd_handle_mapd(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapd()  argument
1216 	if (!vgic_its_check_id(its, its->baser_device_table, device_id, NULL))  in vgic_its_cmd_handle_mapd()
1222 	device = find_its_device(its, device_id);  in vgic_its_cmd_handle_mapd()
1239 	device = vgic_its_alloc_device(its, device_id, itt_addr,  in vgic_its_cmd_handle_mapd()
1249 static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_mapc()  argument
1265 		vgic_its_free_collection(its, coll_id);  in vgic_its_cmd_handle_mapc()
1268 		collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_mapc()
1273 			if (!vgic_its_check_id(its, its->baser_coll_table,  in vgic_its_cmd_handle_mapc()
1277 			ret = vgic_its_alloc_collection(its, &collection,  in vgic_its_cmd_handle_mapc()
1284 			update_affinity_collection(kvm, its, collection);  in vgic_its_cmd_handle_mapc()
1295 static int vgic_its_cmd_handle_clear(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_clear()  argument
1303 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_clear()
1325 static int vgic_its_cmd_handle_inv(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_inv()  argument
1333 	ite = find_ite(its, device_id, event_id);  in vgic_its_cmd_handle_inv()
1382 static int vgic_its_cmd_handle_invall(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_invall()  argument
1389 	collection = find_collection(its, coll_id);  in vgic_its_cmd_handle_invall()
1407 static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_movall()  argument
1451 static int vgic_its_cmd_handle_int(struct kvm *kvm, struct vgic_its *its,  in vgic_its_cmd_handle_int()  argument
1457 	return vgic_its_trigger_msi(kvm, its, msi_devid, msi_data);  in vgic_its_cmd_handle_int()
1464 static int vgic_its_handle_command(struct kvm *kvm, struct vgic_its *its,  in vgic_its_handle_command()  argument
1469 	mutex_lock(&its->its_lock);  in vgic_its_handle_command()
1472 		ret = vgic_its_cmd_handle_mapd(kvm, its, its_cmd);  in vgic_its_handle_command()
1475 		ret = vgic_its_cmd_handle_mapc(kvm, its, its_cmd);  in vgic_its_handle_command()
1478 		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);  in vgic_its_handle_command()
1481 		ret = vgic_its_cmd_handle_mapi(kvm, its, its_cmd);  in vgic_its_handle_command()
1484 		ret = vgic_its_cmd_handle_movi(kvm, its, its_cmd);  in vgic_its_handle_command()
1487 		ret = vgic_its_cmd_handle_discard(kvm, its, its_cmd);  in vgic_its_handle_command()
1490 		ret = vgic_its_cmd_handle_clear(kvm, its, its_cmd);  in vgic_its_handle_command()
1493 		ret = vgic_its_cmd_handle_movall(kvm, its, its_cmd);  in vgic_its_handle_command()
1496 		ret = vgic_its_cmd_handle_int(kvm, its, its_cmd);  in vgic_its_handle_command()
1499 		ret = vgic_its_cmd_handle_inv(kvm, its, its_cmd);  in vgic_its_handle_command()
1502 		ret = vgic_its_cmd_handle_invall(kvm, its, its_cmd);  in vgic_its_handle_command()
1509 	mutex_unlock(&its->its_lock);  in vgic_its_handle_command()
1551 					       struct vgic_its *its,  in vgic_mmio_read_its_cbaser()  argument
1554 	return extract_bytes(its->cbaser, addr & 7, len);  in vgic_mmio_read_its_cbaser()
1557 static void vgic_mmio_write_its_cbaser(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_cbaser()  argument
1562 	if (its->enabled)  in vgic_mmio_write_its_cbaser()
1565 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_cbaser()
1566 	its->cbaser = update_64bit_reg(its->cbaser, addr & 7, len, val);  in vgic_mmio_write_its_cbaser()
1567 	its->cbaser = vgic_sanitise_its_cbaser(its->cbaser);  in vgic_mmio_write_its_cbaser()
1568 	its->creadr = 0;  in vgic_mmio_write_its_cbaser()
1573 	its->cwriter = its->creadr;  in vgic_mmio_write_its_cbaser()
1574 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cbaser()
1582 static void vgic_its_process_commands(struct kvm *kvm, struct vgic_its *its)  in vgic_its_process_commands()  argument
1588 	if (!its->enabled)  in vgic_its_process_commands()
1591 	cbaser = GITS_CBASER_ADDRESS(its->cbaser);  in vgic_its_process_commands()
1593 	while (its->cwriter != its->creadr) {  in vgic_its_process_commands()
1594 		int ret = kvm_read_guest_lock(kvm, cbaser + its->creadr,  in vgic_its_process_commands()
1604 			vgic_its_handle_command(kvm, its, cmd_buf);  in vgic_its_process_commands()
1606 		its->creadr += ITS_CMD_SIZE;  in vgic_its_process_commands()
1607 		if (its->creadr == ITS_CMD_BUFFER_SIZE(its->cbaser))  in vgic_its_process_commands()
1608 			its->creadr = 0;  in vgic_its_process_commands()
1618 static void vgic_mmio_write_its_cwriter(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_cwriter()  argument
1624 	if (!its)  in vgic_mmio_write_its_cwriter()
1627 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1629 	reg = update_64bit_reg(its->cwriter, addr & 7, len, val);  in vgic_mmio_write_its_cwriter()
1631 	if (reg >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {  in vgic_mmio_write_its_cwriter()
1632 		mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1635 	its->cwriter = reg;  in vgic_mmio_write_its_cwriter()
1637 	vgic_its_process_commands(kvm, its);  in vgic_mmio_write_its_cwriter()
1639 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_cwriter()
1643 						struct vgic_its *its,  in vgic_mmio_read_its_cwriter()  argument
1646 	return extract_bytes(its->cwriter, addr & 0x7, len);  in vgic_mmio_read_its_cwriter()
1650 					       struct vgic_its *its,  in vgic_mmio_read_its_creadr()  argument
1653 	return extract_bytes(its->creadr, addr & 0x7, len);  in vgic_mmio_read_its_creadr()
1657 					      struct vgic_its *its,  in vgic_mmio_uaccess_write_its_creadr()  argument
1664 	mutex_lock(&its->cmd_lock);  in vgic_mmio_uaccess_write_its_creadr()
1666 	if (its->enabled) {  in vgic_mmio_uaccess_write_its_creadr()
1672 	if (cmd_offset >= ITS_CMD_BUFFER_SIZE(its->cbaser)) {  in vgic_mmio_uaccess_write_its_creadr()
1677 	its->creadr = cmd_offset;  in vgic_mmio_uaccess_write_its_creadr()
1679 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_uaccess_write_its_creadr()
1685 					      struct vgic_its *its,  in vgic_mmio_read_its_baser()  argument
1692 		reg = its->baser_device_table;  in vgic_mmio_read_its_baser()
1695 		reg = its->baser_coll_table;  in vgic_mmio_read_its_baser()
1707 				      struct vgic_its *its,  in vgic_mmio_write_its_baser()  argument
1711 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_mmio_write_its_baser()
1716 	if (its->enabled)  in vgic_mmio_write_its_baser()
1721 		regptr = &its->baser_device_table;  in vgic_mmio_write_its_baser()
1726 		regptr = &its->baser_coll_table;  in vgic_mmio_write_its_baser()
1747 		mutex_lock(&its->its_lock);  in vgic_mmio_write_its_baser()
1750 			vgic_its_free_device_list(kvm, its);  in vgic_mmio_write_its_baser()
1753 			vgic_its_free_collection_list(kvm, its);  in vgic_mmio_write_its_baser()
1756 		mutex_unlock(&its->its_lock);  in vgic_mmio_write_its_baser()
1761 					     struct vgic_its *its,  in vgic_mmio_read_its_ctlr()  argument
1766 	mutex_lock(&its->cmd_lock);  in vgic_mmio_read_its_ctlr()
1767 	if (its->creadr == its->cwriter)  in vgic_mmio_read_its_ctlr()
1769 	if (its->enabled)  in vgic_mmio_read_its_ctlr()
1771 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_read_its_ctlr()
1776 static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its,  in vgic_mmio_write_its_ctlr()  argument
1780 	mutex_lock(&its->cmd_lock);  in vgic_mmio_write_its_ctlr()
1786 	if (!its->enabled && (val & GITS_CTLR_ENABLE) &&  in vgic_mmio_write_its_ctlr()
1787 		(!(its->baser_device_table & GITS_BASER_VALID) ||  in vgic_mmio_write_its_ctlr()
1788 		 !(its->baser_coll_table & GITS_BASER_VALID) ||  in vgic_mmio_write_its_ctlr()
1789 		 !(its->cbaser & GITS_CBASER_VALID)))  in vgic_mmio_write_its_ctlr()
1792 	its->enabled = !!(val & GITS_CTLR_ENABLE);  in vgic_mmio_write_its_ctlr()
1793 	if (!its->enabled)  in vgic_mmio_write_its_ctlr()
1800 	vgic_its_process_commands(kvm, its);  in vgic_mmio_write_its_ctlr()
1803 	mutex_unlock(&its->cmd_lock);  in vgic_mmio_write_its_ctlr()
1825 static void its_mmio_write_wi(struct kvm *kvm, struct vgic_its *its,  in its_mmio_write_wi()  argument
1867 static int vgic_register_its_iodev(struct kvm *kvm, struct vgic_its *its,  in vgic_register_its_iodev()  argument
1870 	struct vgic_io_device *iodev = &its->iodev;  in vgic_register_its_iodev()
1874 	if (!IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {  in vgic_register_its_iodev()
1879 	its->vgic_its_base = addr;  in vgic_register_its_iodev()
1884 	iodev->base_addr = its->vgic_its_base;  in vgic_register_its_iodev()
1886 	iodev->its = its;  in vgic_register_its_iodev()
1950 	struct vgic_its *its;  in vgic_its_create()  local
1955 	its = kzalloc(sizeof(struct vgic_its), GFP_KERNEL_ACCOUNT);  in vgic_its_create()
1956 	if (!its)  in vgic_its_create()
1965 			kfree(its);  in vgic_its_create()
1972 	mutex_init(&its->its_lock);  in vgic_its_create()
1973 	mutex_init(&its->cmd_lock);  in vgic_its_create()
1977 	mutex_lock(&its->cmd_lock);  in vgic_its_create()
1978 	mutex_lock(&its->its_lock);  in vgic_its_create()
1979 	mutex_unlock(&its->its_lock);  in vgic_its_create()
1980 	mutex_unlock(&its->cmd_lock);  in vgic_its_create()
1983 	its->vgic_its_base = VGIC_ADDR_UNDEF;  in vgic_its_create()
1985 	INIT_LIST_HEAD(&its->device_list);  in vgic_its_create()
1986 	INIT_LIST_HEAD(&its->collection_list);  in vgic_its_create()
1990 	its->enabled = false;  in vgic_its_create()
1991 	its->dev = dev;  in vgic_its_create()
1993 	its->baser_device_table = INITIAL_BASER_VALUE			|  in vgic_its_create()
1995 	its->baser_coll_table = INITIAL_BASER_VALUE |  in vgic_its_create()
1999 	dev->private = its;  in vgic_its_create()
2001 	ret = vgic_its_set_abi(its, NR_ITS_ABIS - 1);  in vgic_its_create()
2011 	struct vgic_its *its = kvm_dev->private;  in vgic_its_destroy()  local
2013 	mutex_lock(&its->its_lock);  in vgic_its_destroy()
2015 	vgic_its_free_device_list(kvm, its);  in vgic_its_destroy()
2016 	vgic_its_free_collection_list(kvm, its);  in vgic_its_destroy()
2018 	mutex_unlock(&its->its_lock);  in vgic_its_destroy()
2019 	kfree(its);  in vgic_its_destroy()
2049 	struct vgic_its *its;  in vgic_its_attr_regs_access()  local
2054 	its = dev->private;  in vgic_its_attr_regs_access()
2081 	if (IS_VGIC_ADDR_UNDEF(its->vgic_its_base)) {  in vgic_its_attr_regs_access()
2094 	addr = its->vgic_its_base + offset;  in vgic_its_attr_regs_access()
2100 			ret = region->uaccess_its_write(dev->kvm, its, addr,  in vgic_its_attr_regs_access()
2103 			region->its_write(dev->kvm, its, addr, len, *reg);  in vgic_its_attr_regs_access()
2105 		*reg = region->its_read(dev->kvm, its, addr, len);  in vgic_its_attr_regs_access()
2151 typedef int (*entry_fn_t)(struct vgic_its *its, u32 id, void *entry,
2169 static int scan_its_table(struct vgic_its *its, gpa_t base, int size, u32 esz,  in scan_its_table()  argument
2172 	struct kvm *kvm = its->dev->kvm;  in scan_its_table()
2189 		next_offset = fn(its, id, entry, opaque);  in scan_its_table()
2207 static int vgic_its_save_ite(struct vgic_its *its, struct its_device *dev,  in vgic_its_save_ite()  argument
2210 	struct kvm *kvm = its->dev->kvm;  in vgic_its_save_ite()
2228 static int vgic_its_restore_ite(struct vgic_its *its, u32 event_id,  in vgic_its_restore_ite()  argument
2233 	struct kvm *kvm = its->dev->kvm;  in vgic_its_restore_ite()
2259 	collection = find_collection(its, coll_id);  in vgic_its_restore_ite()
2263 	if (!vgic_its_check_event_id(its, dev, event_id))  in vgic_its_restore_ite()
2295 static int vgic_its_save_itt(struct vgic_its *its, struct its_device *device)  in vgic_its_save_itt()  argument
2297 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_save_itt()
2317 		ret = vgic_its_save_ite(its, device, ite, gpa, ite_esz);  in vgic_its_save_itt()
2332 static int vgic_its_restore_itt(struct vgic_its *its, struct its_device *dev)  in vgic_its_restore_itt()  argument
2334 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_itt()
2340 	ret = scan_its_table(its, base, max_size, ite_esz, 0,  in vgic_its_restore_itt()
2357 static int vgic_its_save_dte(struct vgic_its *its, struct its_device *dev,  in vgic_its_save_dte()  argument
2360 	struct kvm *kvm = its->dev->kvm;  in vgic_its_save_dte()
2365 	next_offset = compute_next_devid_offset(&its->device_list, dev);  in vgic_its_save_dte()
2385 static int vgic_its_restore_dte(struct vgic_its *its, u32 id,  in vgic_its_restore_dte()  argument
2389 	u64 baser = its->baser_device_table;  in vgic_its_restore_dte()
2410 	if (!vgic_its_check_id(its, baser, id, NULL))  in vgic_its_restore_dte()
2413 	dev = vgic_its_alloc_device(its, id, itt_addr, num_eventid_bits);  in vgic_its_restore_dte()
2417 	ret = vgic_its_restore_itt(its, dev);  in vgic_its_restore_dte()
2419 		vgic_its_free_device(its->dev->kvm, dev);  in vgic_its_restore_dte()
2445 static int vgic_its_save_device_tables(struct vgic_its *its)  in vgic_its_save_device_tables()  argument
2447 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_save_device_tables()
2448 	u64 baser = its->baser_device_table;  in vgic_its_save_device_tables()
2455 	list_sort(NULL, &its->device_list, vgic_its_device_cmp);  in vgic_its_save_device_tables()
2457 	list_for_each_entry(dev, &its->device_list, dev_list) {  in vgic_its_save_device_tables()
2461 		if (!vgic_its_check_id(its, baser,  in vgic_its_save_device_tables()
2465 		ret = vgic_its_save_itt(its, dev);  in vgic_its_save_device_tables()
2469 		ret = vgic_its_save_dte(its, dev, eaddr, dte_esz);  in vgic_its_save_device_tables()
2488 static int handle_l1_dte(struct vgic_its *its, u32 id, void *addr,  in handle_l1_dte()  argument
2491 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in handle_l1_dte()
2505 	ret = scan_its_table(its, gpa, SZ_64K, dte_esz,  in handle_l1_dte()
2515 static int vgic_its_restore_device_tables(struct vgic_its *its)  in vgic_its_restore_device_tables()  argument
2517 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_device_tables()
2518 	u64 baser = its->baser_device_table;  in vgic_its_restore_device_tables()
2530 		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,  in vgic_its_restore_device_tables()
2534 		ret = scan_its_table(its, l1_gpa, l1_tbl_size, l1_esz, 0,  in vgic_its_restore_device_tables()
2543 		vgic_its_free_device_list(its->dev->kvm, its);  in vgic_its_restore_device_tables()
2548 static int vgic_its_save_cte(struct vgic_its *its,  in vgic_its_save_cte()  argument
2558 	return vgic_write_guest_lock(its->dev->kvm, gpa, &val, esz);  in vgic_its_save_cte()
2566 static int vgic_its_restore_cte(struct vgic_its *its, gpa_t gpa, int esz)  in vgic_its_restore_cte()  argument
2569 	struct kvm *kvm = its->dev->kvm;  in vgic_its_restore_cte()
2589 	collection = find_collection(its, coll_id);  in vgic_its_restore_cte()
2593 	if (!vgic_its_check_id(its, its->baser_coll_table, coll_id, NULL))  in vgic_its_restore_cte()
2596 	ret = vgic_its_alloc_collection(its, &collection, coll_id);  in vgic_its_restore_cte()
2607 static int vgic_its_save_collection_table(struct vgic_its *its)  in vgic_its_save_collection_table()  argument
2609 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_save_collection_table()
2610 	u64 baser = its->baser_coll_table;  in vgic_its_save_collection_table()
2622 	list_for_each_entry(collection, &its->collection_list, coll_list) {  in vgic_its_save_collection_table()
2623 		ret = vgic_its_save_cte(its, collection, gpa, cte_esz);  in vgic_its_save_collection_table()
2639 	ret = vgic_write_guest_lock(its->dev->kvm, gpa, &val, cte_esz);  in vgic_its_save_collection_table()
2648 static int vgic_its_restore_collection_table(struct vgic_its *its)  in vgic_its_restore_collection_table()  argument
2650 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_restore_collection_table()
2651 	u64 baser = its->baser_coll_table;  in vgic_its_restore_collection_table()
2665 		ret = vgic_its_restore_cte(its, gpa, cte_esz);  in vgic_its_restore_collection_table()
2676 		vgic_its_free_collection_list(its->dev->kvm, its);  in vgic_its_restore_collection_table()
2685 static int vgic_its_save_tables_v0(struct vgic_its *its)  in vgic_its_save_tables_v0()  argument
2689 	ret = vgic_its_save_device_tables(its);  in vgic_its_save_tables_v0()
2693 	return vgic_its_save_collection_table(its);  in vgic_its_save_tables_v0()
2701 static int vgic_its_restore_tables_v0(struct vgic_its *its)  in vgic_its_restore_tables_v0()  argument
2705 	ret = vgic_its_restore_collection_table(its);  in vgic_its_restore_tables_v0()
2709 	ret = vgic_its_restore_device_tables(its);  in vgic_its_restore_tables_v0()
2711 		vgic_its_free_collection_list(its->dev->kvm, its);  in vgic_its_restore_tables_v0()
2715 static int vgic_its_commit_v0(struct vgic_its *its)  in vgic_its_commit_v0()  argument
2719 	abi = vgic_its_get_abi(its);  in vgic_its_commit_v0()
2720 	its->baser_coll_table &= ~GITS_BASER_ENTRY_SIZE_MASK;  in vgic_its_commit_v0()
2721 	its->baser_device_table &= ~GITS_BASER_ENTRY_SIZE_MASK;  in vgic_its_commit_v0()
2723 	its->baser_coll_table |= (GIC_ENCODE_SZ(abi->cte_esz, 5)  in vgic_its_commit_v0()
2726 	its->baser_device_table |= (GIC_ENCODE_SZ(abi->dte_esz, 5)  in vgic_its_commit_v0()
2731 static void vgic_its_reset(struct kvm *kvm, struct vgic_its *its)  in vgic_its_reset()  argument
2734 	its->baser_coll_table &= ~GITS_BASER_VALID;  in vgic_its_reset()
2735 	its->baser_device_table &= ~GITS_BASER_VALID;  in vgic_its_reset()
2736 	its->cbaser = 0;  in vgic_its_reset()
2737 	its->creadr = 0;  in vgic_its_reset()
2738 	its->cwriter = 0;  in vgic_its_reset()
2739 	its->enabled = 0;  in vgic_its_reset()
2740 	vgic_its_free_device_list(kvm, its);  in vgic_its_reset()
2741 	vgic_its_free_collection_list(kvm, its);  in vgic_its_reset()
2772 static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)  in vgic_its_ctrl()  argument
2774 	const struct vgic_its_abi *abi = vgic_its_get_abi(its);  in vgic_its_ctrl()
2788 	mutex_lock(&its->its_lock);  in vgic_its_ctrl()
2792 		vgic_its_reset(kvm, its);  in vgic_its_ctrl()
2795 		ret = abi->save_tables(its);  in vgic_its_ctrl()
2798 		ret = abi->restore_tables(its);  in vgic_its_ctrl()
2802 	mutex_unlock(&its->its_lock);  in vgic_its_ctrl()
2829 	struct vgic_its *its = dev->private;  in vgic_its_set_attr()  local
2844 		ret = vgic_check_iorange(dev->kvm, its->vgic_its_base,  in vgic_its_set_attr()
2849 		return vgic_register_its_iodev(dev->kvm, its, addr);  in vgic_its_set_attr()
2852 		return vgic_its_ctrl(dev->kvm, its, attr->attr);  in vgic_its_set_attr()
2871 		struct vgic_its *its = dev->private;  in vgic_its_get_attr()  local
2872 		u64 addr = its->vgic_its_base;  in vgic_its_get_attr()