602 #define GPUVA_START(node) ((node)->va.addr)
603 #define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1)
613 			      struct drm_gpuva *va);
614 static void __drm_gpuva_remove(struct drm_gpuva *va);
639 	u64 kstart = mgr->kernel_alloc_node.va.addr;  in drm_gpuva_in_kernel_node()
640 	u64 krange = mgr->kernel_alloc_node.va.range;  in drm_gpuva_in_kernel_node()
690 		mgr->kernel_alloc_node.va.addr = reserve_offset;  in drm_gpuva_manager_init()
691 		mgr->kernel_alloc_node.va.range = reserve_range;  in drm_gpuva_manager_init()
712 	if (mgr->kernel_alloc_node.va.range)  in drm_gpuva_manager_destroy()
722 		   struct drm_gpuva *va)  in __drm_gpuva_insert()  argument
728 				    GPUVA_START(va),  in __drm_gpuva_insert()
729 				    GPUVA_LAST(va)))  in __drm_gpuva_insert()
732 	va->mgr = mgr;  in __drm_gpuva_insert()
734 	drm_gpuva_it_insert(va, &mgr->rb.tree);  in __drm_gpuva_insert()
736 	node = rb_prev(&va->rb.node);  in __drm_gpuva_insert()
742 	list_add(&va->rb.entry, head);  in __drm_gpuva_insert()
763 		 struct drm_gpuva *va)  in drm_gpuva_insert()  argument
765 	u64 addr = va->va.addr;  in drm_gpuva_insert()
766 	u64 range = va->va.range;  in drm_gpuva_insert()
771 	return __drm_gpuva_insert(mgr, va);  in drm_gpuva_insert()
776 __drm_gpuva_remove(struct drm_gpuva *va)  in __drm_gpuva_remove()  argument
778 	drm_gpuva_it_remove(va, &va->mgr->rb.tree);  in __drm_gpuva_remove()
779 	list_del_init(&va->rb.entry);  in __drm_gpuva_remove()
793 drm_gpuva_remove(struct drm_gpuva *va)  in drm_gpuva_remove()  argument
795 	struct drm_gpuva_manager *mgr = va->mgr;  in drm_gpuva_remove()
797 	if (unlikely(va == &mgr->kernel_alloc_node)) {  in drm_gpuva_remove()
802 	__drm_gpuva_remove(va);  in drm_gpuva_remove()
817 drm_gpuva_link(struct drm_gpuva *va)  in drm_gpuva_link()  argument
819 	struct drm_gem_object *obj = va->gem.obj;  in drm_gpuva_link()
826 	list_add_tail(&va->gem.entry, &obj->gpuva.list);  in drm_gpuva_link()
841 drm_gpuva_unlink(struct drm_gpuva *va)  in drm_gpuva_unlink()  argument
843 	struct drm_gem_object *obj = va->gem.obj;  in drm_gpuva_unlink()
850 	list_del_init(&va->gem.entry);  in drm_gpuva_unlink()
884 	struct drm_gpuva *va;  in drm_gpuva_find()  local
886 	va = drm_gpuva_find_first(mgr, addr, range);  in drm_gpuva_find()
887 	if (!va)  in drm_gpuva_find()
890 	if (va->va.addr != addr ||  in drm_gpuva_find()
891 	    va->va.range != range)  in drm_gpuva_find()
894 	return va;  in drm_gpuva_find()
972 	      struct drm_gpuva *va,  in drm_gpuva_map()  argument
975 	drm_gpuva_init_from_op(va, op);  in drm_gpuva_map()
976 	drm_gpuva_insert(mgr, va);  in drm_gpuva_map()
995 	struct drm_gpuva *curr = op->unmap->va;  in drm_gpuva_remap()
1022 	drm_gpuva_remove(op->va);  in drm_gpuva_unmap()
1034 	op.map.va.addr = addr;  in op_map_cb()
1035 	op.map.va.range = range;  in op_map_cb()
1062 	    struct drm_gpuva *va, bool merge)  in op_unmap_cb()  argument
1067 	op.unmap.va = va;  in op_unmap_cb()
1079 	struct drm_gpuva *va, *next;  in __drm_gpuva_sm_map()  local
1086 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {  in __drm_gpuva_sm_map()
1087 		struct drm_gem_object *obj = va->gem.obj;  in __drm_gpuva_sm_map()
1088 		u64 offset = va->gem.offset;  in __drm_gpuva_sm_map()
1089 		u64 addr = va->va.addr;  in __drm_gpuva_sm_map()
1090 		u64 range = va->va.range;  in __drm_gpuva_sm_map()
1092 		bool merge = !!va->gem.obj;  in __drm_gpuva_sm_map()
1099 				ret = op_unmap_cb(ops, priv, va, merge);  in __drm_gpuva_sm_map()
1106 				ret = op_unmap_cb(ops, priv, va, merge);  in __drm_gpuva_sm_map()
1114 					.va.addr = req_end,  in __drm_gpuva_sm_map()
1115 					.va.range = range - req_range,  in __drm_gpuva_sm_map()
1120 					.va = va,  in __drm_gpuva_sm_map()
1132 				.va.addr = addr,  in __drm_gpuva_sm_map()
1133 				.va.range = ls_range,  in __drm_gpuva_sm_map()
1137 			struct drm_gpuva_op_unmap u = { .va = va };  in __drm_gpuva_sm_map()
1159 					.va.addr = req_end,  in __drm_gpuva_sm_map()
1160 					.va.range = end - req_end,  in __drm_gpuva_sm_map()
1177 				ret = op_unmap_cb(ops, priv, va, merge);  in __drm_gpuva_sm_map()
1184 				ret = op_unmap_cb(ops, priv, va, merge);  in __drm_gpuva_sm_map()
1192 					.va.addr = req_end,  in __drm_gpuva_sm_map()
1193 					.va.range = end - req_end,  in __drm_gpuva_sm_map()
1198 					.va = va,  in __drm_gpuva_sm_map()
1220 	struct drm_gpuva *va, *next;  in __drm_gpuva_sm_unmap()  local
1227 	drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) {  in __drm_gpuva_sm_unmap()
1230 		struct drm_gem_object *obj = va->gem.obj;  in __drm_gpuva_sm_unmap()
1231 		u64 offset = va->gem.offset;  in __drm_gpuva_sm_unmap()
1232 		u64 addr = va->va.addr;  in __drm_gpuva_sm_unmap()
1233 		u64 range = va->va.range;  in __drm_gpuva_sm_unmap()
1237 			prev.va.addr = addr;  in __drm_gpuva_sm_unmap()
1238 			prev.va.range = req_addr - addr;  in __drm_gpuva_sm_unmap()
1246 			next.va.addr = req_end;  in __drm_gpuva_sm_unmap()
1247 			next.va.range = end - req_end;  in __drm_gpuva_sm_unmap()
1255 			struct drm_gpuva_op_unmap unmap = { .va = va };  in __drm_gpuva_sm_unmap()
1264 			ret = op_unmap_cb(ops, priv, va, false);  in __drm_gpuva_sm_unmap()
1609 	struct drm_gpuva *va;  in drm_gpuva_prefetch_ops_create()  local
1619 	drm_gpuva_for_each_va_range(va, mgr, addr, end) {  in drm_gpuva_prefetch_ops_create()
1627 		op->prefetch.va = va;  in drm_gpuva_prefetch_ops_create()
1664 	struct drm_gpuva *va;  in drm_gpuva_gem_unmap_ops_create()  local
1675 	drm_gem_for_each_gpuva(va, obj) {  in drm_gpuva_gem_unmap_ops_create()
1683 		op->unmap.va = va;  in drm_gpuva_gem_unmap_ops_create()