198 		     const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)  in nvkm_vmm_unref_sptes()  argument
204 	u32 spti = ptei & (sptn - 1), lpti, pteb;  in nvkm_vmm_unref_sptes()
209 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {  in nvkm_vmm_unref_sptes()
219 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {  in nvkm_vmm_unref_sptes()
222 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_unref_sptes()
223 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_SPTES))  in nvkm_vmm_unref_sptes()
235 		pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;  in nvkm_vmm_unref_sptes()
236 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_unref_sptes()
237 			if (pgt->pte[ptei] & NVKM_VMM_PTE_SPTES)  in nvkm_vmm_unref_sptes()
239 			pgt->pte[ptei] &= ~NVKM_VMM_PTE_VALID;  in nvkm_vmm_unref_sptes()
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_unref_ptes()  argument
267 		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);  in nvkm_vmm_unref_ptes()
272 			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);  in nvkm_vmm_unref_ptes()
281 		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);  in nvkm_vmm_unref_ptes()
297 		   const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)  in nvkm_vmm_ref_sptes()  argument
303 	u32 spti = ptei & (sptn - 1), lpti, pteb;  in nvkm_vmm_ref_sptes()
308 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {  in nvkm_vmm_ref_sptes()
318 	for (ptei = pteb = ptei >> sptb; ptei < lpti; pteb = ptei) {  in nvkm_vmm_ref_sptes()
321 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_ref_sptes()
322 				if (!(pgt->pte[ptei] & NVKM_VMM_PTE_VALID))  in nvkm_vmm_ref_sptes()
334 		pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;  in nvkm_vmm_ref_sptes()
335 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_ref_sptes()
336 			if (pgt->pte[ptei] & NVKM_VMM_PTE_VALID)  in nvkm_vmm_ref_sptes()
338 			pgt->pte[ptei] |= NVKM_VMM_PTE_VALID;  in nvkm_vmm_ref_sptes()
364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_ref_ptes()  argument
375 		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);  in nvkm_vmm_ref_ptes()
382 		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_ptes()  argument
386 			pgt->pde[ptei++] = NVKM_VMM_PDE_SPARSE;  in nvkm_vmm_sparse_ptes()
389 		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);  in nvkm_vmm_sparse_ptes()
394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_unref_ptes()  argument
398 		memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);  in nvkm_vmm_sparse_unref_ptes()
401 		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);  in nvkm_vmm_sparse_unref_ptes()
402 	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);  in nvkm_vmm_sparse_unref_ptes()
406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_ref_ptes()  argument
408 	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);  in nvkm_vmm_sparse_ref_ptes()
409 	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);  in nvkm_vmm_sparse_ref_ptes()
423 	u32 pteb, ptei, ptes;  in nvkm_vmm_ref_hwpt()  local
447 		for (ptei = pteb = 0; ptei < pten; pteb = ptei) {  in nvkm_vmm_ref_hwpt()
448 			bool spte = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;  in nvkm_vmm_ref_hwpt()
449 			for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {  in nvkm_vmm_ref_hwpt()
450 				bool next = pgt->pte[ptei] & NVKM_VMM_PTE_SPTES;  in nvkm_vmm_ref_hwpt()
535 		const u32 ptei = it.pte[0];  in nvkm_vmm_iter()  local
536 		const u32 ptes = min_t(u64, it.cnt, pten - ptei);  in nvkm_vmm_iter()
563 		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {  in nvkm_vmm_iter()
567 					MAP_PTES(vmm, pt, ptei, ptes, map);  in nvkm_vmm_iter()
569 					CLR_PTES(vmm, pt, ptei, ptes);  in nvkm_vmm_iter()
1801 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_boot_ptes()  argument