1 /*
2  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  *
23  * Authors:
24  *    Kevin Tian <kevin.tian@intel.com>
25  *    Eddie Dong <eddie.dong@intel.com>
26  *
27  * Contributors:
28  *    Niu Bing <bing.niu@intel.com>
29  *    Zhi Wang <zhi.a.wang@intel.com>
30  *
31  */
32 
33 #ifndef _GVT_H_
34 #define _GVT_H_
35 
36 #include <uapi/linux/pci_regs.h>
37 #include <linux/vfio.h>
38 #include <linux/mdev.h>
39 
40 #include <asm/kvm_page_track.h>
41 
42 #include "i915_drv.h"
43 #include "intel_gvt.h"
44 
45 #include "debug.h"
46 #include "mmio.h"
47 #include "reg.h"
48 #include "interrupt.h"
49 #include "gtt.h"
50 #include "display.h"
51 #include "edid.h"
52 #include "execlist.h"
53 #include "scheduler.h"
54 #include "sched_policy.h"
55 #include "mmio_context.h"
56 #include "cmd_parser.h"
57 #include "fb_decoder.h"
58 #include "dmabuf.h"
59 #include "page_track.h"
60 
61 #define GVT_MAX_VGPU 8
62 
63 /* Describe per-platform limitations. */
64 struct intel_gvt_device_info {
65 	u32 max_support_vgpus;
66 	u32 cfg_space_size;
67 	u32 mmio_size;
68 	u32 mmio_bar;
69 	unsigned long msi_cap_offset;
70 	u32 gtt_start_offset;
71 	u32 gtt_entry_size;
72 	u32 gtt_entry_size_shift;
73 	int gmadr_bytes_in_cmd;
74 	u32 max_surface_size;
75 };
76 
77 /* GM resources owned by a vGPU */
78 struct intel_vgpu_gm {
79 	u64 aperture_sz;
80 	u64 hidden_sz;
81 	struct drm_mm_node low_gm_node;
82 	struct drm_mm_node high_gm_node;
83 };
84 
85 #define INTEL_GVT_MAX_NUM_FENCES 32
86 
87 /* Fences owned by a vGPU */
88 struct intel_vgpu_fence {
89 	struct i915_fence_reg *regs[INTEL_GVT_MAX_NUM_FENCES];
90 	u32 base;
91 	u32 size;
92 };
93 
94 struct intel_vgpu_mmio {
95 	void *vreg;
96 };
97 
98 #define INTEL_GVT_MAX_BAR_NUM 4
99 
100 struct intel_vgpu_pci_bar {
101 	u64 size;
102 	bool tracked;
103 };
104 
105 struct intel_vgpu_cfg_space {
106 	unsigned char virtual_cfg_space[PCI_CFG_SPACE_EXP_SIZE];
107 	struct intel_vgpu_pci_bar bar[INTEL_GVT_MAX_BAR_NUM];
108 	u32 pmcsr_off;
109 };
110 
111 #define vgpu_cfg_space(vgpu) ((vgpu)->cfg_space.virtual_cfg_space)
112 
113 struct intel_vgpu_irq {
114 	bool irq_warn_once[INTEL_GVT_EVENT_MAX];
115 	DECLARE_BITMAP(flip_done_event[I915_MAX_PIPES],
116 		       INTEL_GVT_EVENT_MAX);
117 };
118 
119 struct intel_vgpu_opregion {
120 	bool mapped;
121 	void *va;
122 	u32 gfn[INTEL_GVT_OPREGION_PAGES];
123 };
124 
125 #define vgpu_opregion(vgpu) (&(vgpu->opregion))
126 
127 struct intel_vgpu_display {
128 	struct intel_vgpu_i2c_edid i2c_edid;
129 	struct intel_vgpu_port ports[I915_MAX_PORTS];
130 	struct intel_vgpu_sbi sbi;
131 	enum port port_num;
132 };
133 
134 struct vgpu_sched_ctl {
135 	int weight;
136 };
137 
138 enum {
139 	INTEL_VGPU_EXECLIST_SUBMISSION = 1,
140 	INTEL_VGPU_GUC_SUBMISSION,
141 };
142 
143 struct intel_vgpu_submission_ops {
144 	const char *name;
145 	int (*init)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
146 	void (*clean)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
147 	void (*reset)(struct intel_vgpu *vgpu, intel_engine_mask_t engine_mask);
148 };
149 
150 struct intel_vgpu_submission {
151 	struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
152 	struct list_head workload_q_head[I915_NUM_ENGINES];
153 	struct intel_context *shadow[I915_NUM_ENGINES];
154 	struct kmem_cache *workloads;
155 	atomic_t running_workload_num;
156 	union {
157 		u64 i915_context_pml4;
158 		u64 i915_context_pdps[GEN8_3LVL_PDPES];
159 	};
160 	DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
161 	DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
162 	void *ring_scan_buffer[I915_NUM_ENGINES];
163 	int ring_scan_buffer_size[I915_NUM_ENGINES];
164 	const struct intel_vgpu_submission_ops *ops;
165 	int virtual_submission_interface;
166 	bool active;
167 	struct {
168 		u32 lrca;
169 		bool valid;
170 		u64 ring_context_gpa;
171 	} last_ctx[I915_NUM_ENGINES];
172 };
173 
174 #define KVMGT_DEBUGFS_FILENAME		"kvmgt_nr_cache_entries"
175 
176 enum {
177 	INTEL_VGPU_STATUS_ATTACHED = 0,
178 	INTEL_VGPU_STATUS_ACTIVE,
179 	INTEL_VGPU_STATUS_NR_BITS,
180 };
181 
182 struct intel_vgpu {
183 	struct vfio_device vfio_device;
184 	struct intel_gvt *gvt;
185 	struct mutex vgpu_lock;
186 	int id;
187 	DECLARE_BITMAP(status, INTEL_VGPU_STATUS_NR_BITS);
188 	bool pv_notified;
189 	bool failsafe;
190 	unsigned int resetting_eng;
191 
192 	/* Both sched_data and sched_ctl can be seen a part of the global gvt
193 	 * scheduler structure. So below 2 vgpu data are protected
194 	 * by sched_lock, not vgpu_lock.
195 	 */
196 	void *sched_data;
197 	struct vgpu_sched_ctl sched_ctl;
198 
199 	struct intel_vgpu_fence fence;
200 	struct intel_vgpu_gm gm;
201 	struct intel_vgpu_cfg_space cfg_space;
202 	struct intel_vgpu_mmio mmio;
203 	struct intel_vgpu_irq irq;
204 	struct intel_vgpu_gtt gtt;
205 	struct intel_vgpu_opregion opregion;
206 	struct intel_vgpu_display display;
207 	struct intel_vgpu_submission submission;
208 	struct radix_tree_root page_track_tree;
209 	u32 hws_pga[I915_NUM_ENGINES];
210 	/* Set on PCI_D3, reset on DMLR, not reflecting the actual PM state */
211 	bool d3_entered;
212 
213 	struct dentry *debugfs;
214 
215 	struct list_head dmabuf_obj_list_head;
216 	struct mutex dmabuf_lock;
217 	struct idr object_idr;
218 	struct intel_vgpu_vblank_timer vblank_timer;
219 
220 	u32 scan_nonprivbb;
221 
222 	struct vfio_region *region;
223 	int num_regions;
224 	struct eventfd_ctx *intx_trigger;
225 	struct eventfd_ctx *msi_trigger;
226 
227 	/*
228 	 * Two caches are used to avoid mapping duplicated pages (eg.
229 	 * scratch pages). This help to reduce dma setup overhead.
230 	 */
231 	struct rb_root gfn_cache;
232 	struct rb_root dma_addr_cache;
233 	unsigned long nr_cache_entries;
234 	struct mutex cache_lock;
235 
236 	struct kvm_page_track_notifier_node track_node;
237 #define NR_BKT (1 << 18)
238 	struct hlist_head ptable[NR_BKT];
239 #undef NR_BKT
240 };
241 
242 /* validating GM healthy status*/
243 #define vgpu_is_vm_unhealthy(ret_val) \
244 	(((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT))
245 
246 struct intel_gvt_gm {
247 	unsigned long vgpu_allocated_low_gm_size;
248 	unsigned long vgpu_allocated_high_gm_size;
249 };
250 
251 struct intel_gvt_fence {
252 	unsigned long vgpu_allocated_fence_num;
253 };
254 
255 /* Special MMIO blocks. */
256 struct gvt_mmio_block {
257 	unsigned int device;
258 	i915_reg_t   offset;
259 	unsigned int size;
260 	gvt_mmio_func read;
261 	gvt_mmio_func write;
262 };
263 
264 #define INTEL_GVT_MMIO_HASH_BITS 11
265 
266 struct intel_gvt_mmio {
267 	u16 *mmio_attribute;
268 /* Register contains RO bits */
269 #define F_RO		(1 << 0)
270 /* Register contains graphics address */
271 #define F_GMADR		(1 << 1)
272 /* Mode mask registers with high 16 bits as the mask bits */
273 #define F_MODE_MASK	(1 << 2)
274 /* This reg can be accessed by GPU commands */
275 #define F_CMD_ACCESS	(1 << 3)
276 /* This reg has been accessed by a VM */
277 #define F_ACCESSED	(1 << 4)
278 /* This reg requires save & restore during host PM suspend/resume */
279 #define F_PM_SAVE	(1 << 5)
280 /* This reg could be accessed by unaligned address */
281 #define F_UNALIGN	(1 << 6)
282 /* This reg is in GVT's mmio save-restor list and in hardware
283  * logical context image
284  */
285 #define F_SR_IN_CTX	(1 << 7)
286 /* Value of command write of this reg needs to be patched */
287 #define F_CMD_WRITE_PATCH	(1 << 8)
288 
289 	struct gvt_mmio_block *mmio_block;
290 	unsigned int num_mmio_block;
291 
292 	DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
293 	unsigned long num_tracked_mmio;
294 };
295 
296 struct intel_gvt_firmware {
297 	void *cfg_space;
298 	void *mmio;
299 	bool firmware_loaded;
300 };
301 
302 struct intel_vgpu_config {
303 	unsigned int low_mm;
304 	unsigned int high_mm;
305 	unsigned int fence;
306 
307 	/*
308 	 * A vGPU with a weight of 8 will get twice as much GPU as a vGPU with
309 	 * a weight of 4 on a contended host, different vGPU type has different
310 	 * weight set. Legal weights range from 1 to 16.
311 	 */
312 	unsigned int weight;
313 	enum intel_vgpu_edid edid;
314 	const char *name;
315 };
316 
317 struct intel_vgpu_type {
318 	struct mdev_type type;
319 	char name[16];
320 	const struct intel_vgpu_config *conf;
321 };
322 
323 struct intel_gvt {
324 	/* GVT scope lock, protect GVT itself, and all resource currently
325 	 * not yet protected by special locks(vgpu and scheduler lock).
326 	 */
327 	struct mutex lock;
328 	/* scheduler scope lock, protect gvt and vgpu schedule related data */
329 	struct mutex sched_lock;
330 
331 	struct intel_gt *gt;
332 	struct idr vgpu_idr;	/* vGPU IDR pool */
333 
334 	struct intel_gvt_device_info device_info;
335 	struct intel_gvt_gm gm;
336 	struct intel_gvt_fence fence;
337 	struct intel_gvt_mmio mmio;
338 	struct intel_gvt_firmware firmware;
339 	struct intel_gvt_irq irq;
340 	struct intel_gvt_gtt gtt;
341 	struct intel_gvt_workload_scheduler scheduler;
342 	struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES];
343 	DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
344 	struct mdev_parent parent;
345 	struct mdev_type **mdev_types;
346 	struct intel_vgpu_type *types;
347 	unsigned int num_types;
348 	struct intel_vgpu *idle_vgpu;
349 
350 	struct task_struct *service_thread;
351 	wait_queue_head_t service_thread_wq;
352 
353 	/* service_request is always used in bit operation, we should always
354 	 * use it with atomic bit ops so that no need to use gvt big lock.
355 	 */
356 	unsigned long service_request;
357 
358 	struct {
359 		struct engine_mmio *mmio;
360 		int ctx_mmio_count[I915_NUM_ENGINES];
361 		u32 *tlb_mmio_offset_list;
362 		u32 tlb_mmio_offset_list_cnt;
363 		u32 *mocs_mmio_offset_list;
364 		u32 mocs_mmio_offset_list_cnt;
365 	} engine_mmio_list;
366 	bool is_reg_whitelist_updated;
367 
368 	struct dentry *debugfs_root;
369 };
370 
to_gvt(struct drm_i915_private * i915)371 static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
372 {
373 	return i915->gvt;
374 }
375 
376 enum {
377 	/* Scheduling trigger by timer */
378 	INTEL_GVT_REQUEST_SCHED = 0,
379 
380 	/* Scheduling trigger by event */
381 	INTEL_GVT_REQUEST_EVENT_SCHED = 1,
382 
383 	/* per-vGPU vblank emulation request */
384 	INTEL_GVT_REQUEST_EMULATE_VBLANK = 2,
385 	INTEL_GVT_REQUEST_EMULATE_VBLANK_MAX = INTEL_GVT_REQUEST_EMULATE_VBLANK
386 		+ GVT_MAX_VGPU,
387 };
388 
intel_gvt_request_service(struct intel_gvt * gvt,int service)389 static inline void intel_gvt_request_service(struct intel_gvt *gvt,
390 		int service)
391 {
392 	set_bit(service, (void *)&gvt->service_request);
393 	wake_up(&gvt->service_thread_wq);
394 }
395 
396 void intel_gvt_free_firmware(struct intel_gvt *gvt);
397 int intel_gvt_load_firmware(struct intel_gvt *gvt);
398 
399 /* Aperture/GM space definitions for GVT device */
400 #define MB_TO_BYTES(mb) ((mb) << 20ULL)
401 #define BYTES_TO_MB(b) ((b) >> 20ULL)
402 
403 #define HOST_LOW_GM_SIZE MB_TO_BYTES(128)
404 #define HOST_HIGH_GM_SIZE MB_TO_BYTES(384)
405 #define HOST_FENCE 4
406 
407 #define gvt_to_ggtt(gvt)	((gvt)->gt->ggtt)
408 
409 /* Aperture/GM space definitions for GVT device */
410 #define gvt_aperture_sz(gvt)	  gvt_to_ggtt(gvt)->mappable_end
411 #define gvt_aperture_pa_base(gvt) gvt_to_ggtt(gvt)->gmadr.start
412 
413 #define gvt_ggtt_gm_sz(gvt)	gvt_to_ggtt(gvt)->vm.total
414 #define gvt_ggtt_sz(gvt)	(gvt_to_ggtt(gvt)->vm.total >> PAGE_SHIFT << 3)
415 #define gvt_hidden_sz(gvt)	(gvt_ggtt_gm_sz(gvt) - gvt_aperture_sz(gvt))
416 
417 #define gvt_aperture_gmadr_base(gvt) (0)
418 #define gvt_aperture_gmadr_end(gvt) (gvt_aperture_gmadr_base(gvt) \
419 				     + gvt_aperture_sz(gvt) - 1)
420 
421 #define gvt_hidden_gmadr_base(gvt) (gvt_aperture_gmadr_base(gvt) \
422 				    + gvt_aperture_sz(gvt))
423 #define gvt_hidden_gmadr_end(gvt) (gvt_hidden_gmadr_base(gvt) \
424 				   + gvt_hidden_sz(gvt) - 1)
425 
426 #define gvt_fence_sz(gvt) (gvt_to_ggtt(gvt)->num_fences)
427 
428 /* Aperture/GM space definitions for vGPU */
429 #define vgpu_aperture_offset(vgpu)	((vgpu)->gm.low_gm_node.start)
430 #define vgpu_hidden_offset(vgpu)	((vgpu)->gm.high_gm_node.start)
431 #define vgpu_aperture_sz(vgpu)		((vgpu)->gm.aperture_sz)
432 #define vgpu_hidden_sz(vgpu)		((vgpu)->gm.hidden_sz)
433 
434 #define vgpu_aperture_pa_base(vgpu) \
435 	(gvt_aperture_pa_base(vgpu->gvt) + vgpu_aperture_offset(vgpu))
436 
437 #define vgpu_ggtt_gm_sz(vgpu) ((vgpu)->gm.aperture_sz + (vgpu)->gm.hidden_sz)
438 
439 #define vgpu_aperture_pa_end(vgpu) \
440 	(vgpu_aperture_pa_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
441 
442 #define vgpu_aperture_gmadr_base(vgpu) (vgpu_aperture_offset(vgpu))
443 #define vgpu_aperture_gmadr_end(vgpu) \
444 	(vgpu_aperture_gmadr_base(vgpu) + vgpu_aperture_sz(vgpu) - 1)
445 
446 #define vgpu_hidden_gmadr_base(vgpu) (vgpu_hidden_offset(vgpu))
447 #define vgpu_hidden_gmadr_end(vgpu) \
448 	(vgpu_hidden_gmadr_base(vgpu) + vgpu_hidden_sz(vgpu) - 1)
449 
450 #define vgpu_fence_base(vgpu) (vgpu->fence.base)
451 #define vgpu_fence_sz(vgpu) (vgpu->fence.size)
452 
453 /* ring context size i.e. the first 0x50 dwords*/
454 #define RING_CTX_SIZE 320
455 
456 int intel_vgpu_alloc_resource(struct intel_vgpu *vgpu,
457 			      const struct intel_vgpu_config *conf);
458 void intel_vgpu_reset_resource(struct intel_vgpu *vgpu);
459 void intel_vgpu_free_resource(struct intel_vgpu *vgpu);
460 void intel_vgpu_write_fence(struct intel_vgpu *vgpu,
461 	u32 fence, u64 value);
462 
463 /* Macros for easily accessing vGPU virtual/shadow register.
464    Explicitly seperate use for typed MMIO reg or real offset.*/
465 #define vgpu_vreg_t(vgpu, reg) \
466 	(*(u32 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
467 #define vgpu_vreg(vgpu, offset) \
468 	(*(u32 *)(vgpu->mmio.vreg + (offset)))
469 #define vgpu_vreg64_t(vgpu, reg) \
470 	(*(u64 *)(vgpu->mmio.vreg + i915_mmio_reg_offset(reg)))
471 #define vgpu_vreg64(vgpu, offset) \
472 	(*(u64 *)(vgpu->mmio.vreg + (offset)))
473 
474 #define for_each_active_vgpu(gvt, vgpu, id) \
475 	idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) \
476 		for_each_if(test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status))
477 
intel_vgpu_write_pci_bar(struct intel_vgpu * vgpu,u32 offset,u32 val,bool low)478 static inline void intel_vgpu_write_pci_bar(struct intel_vgpu *vgpu,
479 					    u32 offset, u32 val, bool low)
480 {
481 	u32 *pval;
482 
483 	/* BAR offset should be 32 bits algiend */
484 	offset = rounddown(offset, 4);
485 	pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
486 
487 	if (low) {
488 		/*
489 		 * only update bit 31 - bit 4,
490 		 * leave the bit 3 - bit 0 unchanged.
491 		 */
492 		*pval = (val & GENMASK(31, 4)) | (*pval & GENMASK(3, 0));
493 	} else {
494 		*pval = val;
495 	}
496 }
497 
498 int intel_gvt_init_vgpu_types(struct intel_gvt *gvt);
499 void intel_gvt_clean_vgpu_types(struct intel_gvt *gvt);
500 
501 struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt);
502 void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu);
503 int intel_gvt_create_vgpu(struct intel_vgpu *vgpu,
504 			  const struct intel_vgpu_config *conf);
505 void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu);
506 void intel_gvt_release_vgpu(struct intel_vgpu *vgpu);
507 void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
508 				 intel_engine_mask_t engine_mask);
509 void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu);
510 void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu);
511 void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu);
512 
513 int intel_gvt_set_opregion(struct intel_vgpu *vgpu);
514 int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num);
515 
516 /* validating GM functions */
517 #define vgpu_gmadr_is_aperture(vgpu, gmadr) \
518 	((gmadr >= vgpu_aperture_gmadr_base(vgpu)) && \
519 	 (gmadr <= vgpu_aperture_gmadr_end(vgpu)))
520 
521 #define vgpu_gmadr_is_hidden(vgpu, gmadr) \
522 	((gmadr >= vgpu_hidden_gmadr_base(vgpu)) && \
523 	 (gmadr <= vgpu_hidden_gmadr_end(vgpu)))
524 
525 #define vgpu_gmadr_is_valid(vgpu, gmadr) \
526 	 ((vgpu_gmadr_is_aperture(vgpu, gmadr) || \
527 	  (vgpu_gmadr_is_hidden(vgpu, gmadr))))
528 
529 #define gvt_gmadr_is_aperture(gvt, gmadr) \
530 	 ((gmadr >= gvt_aperture_gmadr_base(gvt)) && \
531 	  (gmadr <= gvt_aperture_gmadr_end(gvt)))
532 
533 #define gvt_gmadr_is_hidden(gvt, gmadr) \
534 	  ((gmadr >= gvt_hidden_gmadr_base(gvt)) && \
535 	   (gmadr <= gvt_hidden_gmadr_end(gvt)))
536 
537 #define gvt_gmadr_is_valid(gvt, gmadr) \
538 	  (gvt_gmadr_is_aperture(gvt, gmadr) || \
539 	    gvt_gmadr_is_hidden(gvt, gmadr))
540 
541 bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size);
542 int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr);
543 int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr);
544 int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
545 			     unsigned long *h_index);
546 int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
547 			     unsigned long *g_index);
548 
549 void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
550 		bool primary);
551 void intel_vgpu_reset_cfg_space(struct intel_vgpu *vgpu);
552 
553 int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset,
554 		void *p_data, unsigned int bytes);
555 
556 int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
557 		void *p_data, unsigned int bytes);
558 
559 void intel_vgpu_emulate_hotplug(struct intel_vgpu *vgpu, bool connected);
560 
intel_vgpu_get_bar_gpa(struct intel_vgpu * vgpu,int bar)561 static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
562 {
563 	/* We are 64bit bar. */
564 	return (*(u64 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
565 			PCI_BASE_ADDRESS_MEM_MASK;
566 }
567 
568 void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
569 int intel_vgpu_init_opregion(struct intel_vgpu *vgpu);
570 int intel_vgpu_opregion_base_write_handler(struct intel_vgpu *vgpu, u32 gpa);
571 
572 int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
573 void populate_pvinfo_page(struct intel_vgpu *vgpu);
574 
575 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
576 void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason);
577 void intel_vgpu_detach_regions(struct intel_vgpu *vgpu);
578 
579 enum {
580 	GVT_FAILSAFE_UNSUPPORTED_GUEST,
581 	GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
582 	GVT_FAILSAFE_GUEST_ERR,
583 };
584 
mmio_hw_access_pre(struct intel_gt * gt)585 static inline void mmio_hw_access_pre(struct intel_gt *gt)
586 {
587 	intel_runtime_pm_get(gt->uncore->rpm);
588 }
589 
mmio_hw_access_post(struct intel_gt * gt)590 static inline void mmio_hw_access_post(struct intel_gt *gt)
591 {
592 	intel_runtime_pm_put_unchecked(gt->uncore->rpm);
593 }
594 
595 /**
596  * intel_gvt_mmio_set_accessed - mark a MMIO has been accessed
597  * @gvt: a GVT device
598  * @offset: register offset
599  *
600  */
intel_gvt_mmio_set_accessed(struct intel_gvt * gvt,unsigned int offset)601 static inline void intel_gvt_mmio_set_accessed(
602 			struct intel_gvt *gvt, unsigned int offset)
603 {
604 	gvt->mmio.mmio_attribute[offset >> 2] |= F_ACCESSED;
605 }
606 
607 /**
608  * intel_gvt_mmio_is_cmd_accessible - if a MMIO could be accessed by command
609  * @gvt: a GVT device
610  * @offset: register offset
611  *
612  * Returns:
613  * True if an MMIO is able to be accessed by GPU commands
614  */
intel_gvt_mmio_is_cmd_accessible(struct intel_gvt * gvt,unsigned int offset)615 static inline bool intel_gvt_mmio_is_cmd_accessible(
616 			struct intel_gvt *gvt, unsigned int offset)
617 {
618 	return gvt->mmio.mmio_attribute[offset >> 2] & F_CMD_ACCESS;
619 }
620 
621 /**
622  * intel_gvt_mmio_set_cmd_accessible -
623  *				mark a MMIO could be accessible by command
624  * @gvt: a GVT device
625  * @offset: register offset
626  *
627  */
intel_gvt_mmio_set_cmd_accessible(struct intel_gvt * gvt,unsigned int offset)628 static inline void intel_gvt_mmio_set_cmd_accessible(
629 			struct intel_gvt *gvt, unsigned int offset)
630 {
631 	gvt->mmio.mmio_attribute[offset >> 2] |= F_CMD_ACCESS;
632 }
633 
634 /**
635  * intel_gvt_mmio_is_unalign - mark a MMIO could be accessed unaligned
636  * @gvt: a GVT device
637  * @offset: register offset
638  *
639  */
intel_gvt_mmio_is_unalign(struct intel_gvt * gvt,unsigned int offset)640 static inline bool intel_gvt_mmio_is_unalign(
641 			struct intel_gvt *gvt, unsigned int offset)
642 {
643 	return gvt->mmio.mmio_attribute[offset >> 2] & F_UNALIGN;
644 }
645 
646 /**
647  * intel_gvt_mmio_has_mode_mask - if a MMIO has a mode mask
648  * @gvt: a GVT device
649  * @offset: register offset
650  *
651  * Returns:
652  * True if a MMIO has a mode mask in its higher 16 bits, false if it isn't.
653  *
654  */
intel_gvt_mmio_has_mode_mask(struct intel_gvt * gvt,unsigned int offset)655 static inline bool intel_gvt_mmio_has_mode_mask(
656 			struct intel_gvt *gvt, unsigned int offset)
657 {
658 	return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
659 }
660 
661 /**
662  * intel_gvt_mmio_is_sr_in_ctx -
663  *		check if an MMIO has F_SR_IN_CTX mask
664  * @gvt: a GVT device
665  * @offset: register offset
666  *
667  * Returns:
668  * True if an MMIO has an F_SR_IN_CTX  mask, false if it isn't.
669  *
670  */
intel_gvt_mmio_is_sr_in_ctx(struct intel_gvt * gvt,unsigned int offset)671 static inline bool intel_gvt_mmio_is_sr_in_ctx(
672 			struct intel_gvt *gvt, unsigned int offset)
673 {
674 	return gvt->mmio.mmio_attribute[offset >> 2] & F_SR_IN_CTX;
675 }
676 
677 /**
678  * intel_gvt_mmio_set_sr_in_ctx -
679  *		mask an MMIO in GVT's mmio save-restore list and also
680  *		in hardware logical context image
681  * @gvt: a GVT device
682  * @offset: register offset
683  *
684  */
intel_gvt_mmio_set_sr_in_ctx(struct intel_gvt * gvt,unsigned int offset)685 static inline void intel_gvt_mmio_set_sr_in_ctx(
686 			struct intel_gvt *gvt, unsigned int offset)
687 {
688 	gvt->mmio.mmio_attribute[offset >> 2] |= F_SR_IN_CTX;
689 }
690 
691 void intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu);
692 /**
693  * intel_gvt_mmio_set_cmd_write_patch -
694  *				mark an MMIO if its cmd write needs to be
695  *				patched
696  * @gvt: a GVT device
697  * @offset: register offset
698  *
699  */
intel_gvt_mmio_set_cmd_write_patch(struct intel_gvt * gvt,unsigned int offset)700 static inline void intel_gvt_mmio_set_cmd_write_patch(
701 			struct intel_gvt *gvt, unsigned int offset)
702 {
703 	gvt->mmio.mmio_attribute[offset >> 2] |= F_CMD_WRITE_PATCH;
704 }
705 
706 /**
707  * intel_gvt_mmio_is_cmd_write_patch - check if an mmio's cmd access needs to
708  * be patched
709  * @gvt: a GVT device
710  * @offset: register offset
711  *
712  * Returns:
713  * True if GPU commmand write to an MMIO should be patched
714  */
intel_gvt_mmio_is_cmd_write_patch(struct intel_gvt * gvt,unsigned int offset)715 static inline bool intel_gvt_mmio_is_cmd_write_patch(
716 			struct intel_gvt *gvt, unsigned int offset)
717 {
718 	return gvt->mmio.mmio_attribute[offset >> 2] & F_CMD_WRITE_PATCH;
719 }
720 
721 /**
722  * intel_gvt_read_gpa - copy data from GPA to host data buffer
723  * @vgpu: a vGPU
724  * @gpa: guest physical address
725  * @buf: host data buffer
726  * @len: data length
727  *
728  * Returns:
729  * Zero on success, negative error code if failed.
730  */
intel_gvt_read_gpa(struct intel_vgpu * vgpu,unsigned long gpa,void * buf,unsigned long len)731 static inline int intel_gvt_read_gpa(struct intel_vgpu *vgpu, unsigned long gpa,
732 		void *buf, unsigned long len)
733 {
734 	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
735 		return -ESRCH;
736 	return vfio_dma_rw(&vgpu->vfio_device, gpa, buf, len, false);
737 }
738 
739 /**
740  * intel_gvt_write_gpa - copy data from host data buffer to GPA
741  * @vgpu: a vGPU
742  * @gpa: guest physical address
743  * @buf: host data buffer
744  * @len: data length
745  *
746  * Returns:
747  * Zero on success, negative error code if failed.
748  */
intel_gvt_write_gpa(struct intel_vgpu * vgpu,unsigned long gpa,void * buf,unsigned long len)749 static inline int intel_gvt_write_gpa(struct intel_vgpu *vgpu,
750 		unsigned long gpa, void *buf, unsigned long len)
751 {
752 	if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
753 		return -ESRCH;
754 	return vfio_dma_rw(&vgpu->vfio_device, gpa, buf, len, true);
755 }
756 
757 void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu);
758 void intel_gvt_debugfs_init(struct intel_gvt *gvt);
759 void intel_gvt_debugfs_clean(struct intel_gvt *gvt);
760 
761 int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn);
762 int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn);
763 int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr);
764 int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
765 		unsigned long size, dma_addr_t *dma_addr);
766 void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu,
767 		dma_addr_t dma_addr);
768 
769 #include "trace.h"
770 
771 #endif
772