343 				u64 old_spte, u64 new_spte, int level,
346 static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level)  in handle_changed_spte_acc_track()  argument
348 	if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level))  in handle_changed_spte_acc_track()
351 	if (is_accessed_spte(old_spte) &&  in handle_changed_spte_acc_track()
353 	     spte_to_pfn(old_spte) != spte_to_pfn(new_spte)))  in handle_changed_spte_acc_track()
354 		kvm_set_pfn_accessed(spte_to_pfn(old_spte));  in handle_changed_spte_acc_track()
358 					  u64 old_spte, u64 new_spte, int level)  in handle_changed_spte_dirty_log()  argument
366 	pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);  in handle_changed_spte_dirty_log()
368 	if ((!is_writable_pte(old_spte) || pfn_changed) &&  in handle_changed_spte_dirty_log()
442 		u64 old_spte;  in handle_removed_pt()  local
454 				old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, REMOVED_SPTE);  in handle_removed_pt()
455 				if (!is_removed_spte(old_spte))  in handle_removed_pt()
469 			old_spte = kvm_tdp_mmu_read_spte(sptep);  in handle_removed_pt()
470 			if (!is_shadow_present_pte(old_spte))  in handle_removed_pt()
501 			old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte,  in handle_removed_pt()
505 				    old_spte, REMOVED_SPTE, level, shared);  in handle_removed_pt()
527 				  u64 old_spte, u64 new_spte, int level,  in __handle_changed_spte()  argument
530 	bool was_present = is_shadow_present_pte(old_spte);  in __handle_changed_spte()
532 	bool was_leaf = was_present && is_last_spte(old_spte, level);  in __handle_changed_spte()
534 	bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);  in __handle_changed_spte()
554 		       as_id, gfn, old_spte, new_spte, level);  in __handle_changed_spte()
563 	if (old_spte == new_spte)  in __handle_changed_spte()
566 	trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte);  in __handle_changed_spte()
583 		if (WARN_ON(!is_mmio_spte(old_spte) &&  in __handle_changed_spte()
592 			       as_id, gfn, old_spte, new_spte, level);  in __handle_changed_spte()
599 	if (was_leaf && is_dirty_spte(old_spte) &&  in __handle_changed_spte()
601 		kvm_set_pfn_dirty(spte_to_pfn(old_spte));  in __handle_changed_spte()
611 		handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared);  in __handle_changed_spte()
615 				u64 old_spte, u64 new_spte, int level,  in handle_changed_spte()  argument
618 	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level,  in handle_changed_spte()
620 	handle_changed_spte_acc_track(old_spte, new_spte, level);  in handle_changed_spte()
621 	handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,  in handle_changed_spte()
654 	WARN_ON_ONCE(iter->yielded || is_removed_spte(iter->old_spte));  in tdp_mmu_set_spte_atomic()
662 	if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))  in tdp_mmu_set_spte_atomic()
665 	__handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte,  in tdp_mmu_set_spte_atomic()
667 	handle_changed_spte_acc_track(iter->old_spte, new_spte, iter->level);  in tdp_mmu_set_spte_atomic()
727 			      u64 old_spte, u64 new_spte, gfn_t gfn, int level,  in __tdp_mmu_set_spte()  argument
739 	WARN_ON(is_removed_spte(old_spte) || is_removed_spte(new_spte));  in __tdp_mmu_set_spte()
741 	old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level);  in __tdp_mmu_set_spte()
743 	__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false);  in __tdp_mmu_set_spte()
746 		handle_changed_spte_acc_track(old_spte, new_spte, level);  in __tdp_mmu_set_spte()
748 		handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,  in __tdp_mmu_set_spte()
750 	return old_spte;  in __tdp_mmu_set_spte()
759 	iter->old_spte = __tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep,  in _tdp_mmu_set_spte()
760 					    iter->old_spte, new_spte,  in _tdp_mmu_set_spte()
790 		if (!is_shadow_present_pte(_iter.old_spte) ||		\
791 		    !is_last_spte(_iter.old_spte, _iter.level))		\
867 		if (!is_shadow_present_pte(iter.old_spte))  in __tdp_mmu_zap_root()
918 	u64 old_spte;  in kvm_tdp_mmu_zap_sp()  local
927 	old_spte = kvm_tdp_mmu_read_spte(sp->ptep);  in kvm_tdp_mmu_zap_sp()
928 	if (WARN_ON_ONCE(!is_shadow_present_pte(old_spte)))  in kvm_tdp_mmu_zap_sp()
931 	__tdp_mmu_set_spte(kvm, kvm_mmu_page_as_id(sp), sp->ptep, old_spte, 0,  in kvm_tdp_mmu_zap_sp()
962 		if (!is_shadow_present_pte(iter.old_spte) ||  in tdp_mmu_zap_leafs()
963 		    !is_last_spte(iter.old_spte, iter.level))  in tdp_mmu_zap_leafs()
1076 					 fault->pfn, iter->old_spte, fault->prefetch, true,  in tdp_mmu_map_handle_target_level()
1079 	if (new_spte == iter->old_spte)  in tdp_mmu_map_handle_target_level()
1083 	else if (is_shadow_present_pte(iter->old_spte) &&  in tdp_mmu_map_handle_target_level()
1084 		 !is_last_spte(iter->old_spte, iter->level))  in tdp_mmu_map_handle_target_level()
1170 			disallowed_hugepage_adjust(fault, iter.old_spte, iter.level);  in kvm_tdp_mmu_map()
1180 		if (is_shadow_present_pte(iter.old_spte) &&  in kvm_tdp_mmu_map()
1181 		    is_large_pte(iter.old_spte)) {  in kvm_tdp_mmu_map()
1190 			iter.old_spte = kvm_tdp_mmu_read_spte(iter.sptep);  in kvm_tdp_mmu_map()
1193 		if (!is_shadow_present_pte(iter.old_spte)) {  in kvm_tdp_mmu_map()
1202 			if (is_removed_spte(iter.old_spte))  in kvm_tdp_mmu_map()
1219 	if (iter.level != fault->goal_level || is_removed_spte(iter.old_spte)) {  in kvm_tdp_mmu_map()
1274 	if (!is_accessed_spte(iter->old_spte))  in age_gfn_range()
1277 	new_spte = iter->old_spte;  in age_gfn_range()
1305 	return is_accessed_spte(iter->old_spte);  in test_age_gfn()
1322 	    !is_shadow_present_pte(iter->old_spte))  in set_spte_gfn()
1334 		new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte,  in set_spte_gfn()
1380 		if (!is_shadow_present_pte(iter.old_spte) ||  in wrprot_gfn_range()
1381 		    !is_last_spte(iter.old_spte, iter.level) ||  in wrprot_gfn_range()
1382 		    !(iter.old_spte & PT_WRITABLE_MASK))  in wrprot_gfn_range()
1385 		new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in wrprot_gfn_range()
1478 	const u64 huge_spte = iter->old_spte;  in tdp_mmu_split_huge_page()
1542 		if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte))  in tdp_mmu_split_huge_pages_root()
1550 							      iter.old_spte,  in tdp_mmu_split_huge_pages_root()
1622 		if (!is_shadow_present_pte(iter.old_spte))  in clear_dirty_gfn_range()
1625 		if (spte_ad_need_write_protect(iter.old_spte)) {  in clear_dirty_gfn_range()
1626 			if (is_writable_pte(iter.old_spte))  in clear_dirty_gfn_range()
1627 				new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in clear_dirty_gfn_range()
1631 			if (iter.old_spte & shadow_dirty_mask)  in clear_dirty_gfn_range()
1632 				new_spte = iter.old_spte & ~shadow_dirty_mask;  in clear_dirty_gfn_range()
1695 		if (wrprot || spte_ad_need_write_protect(iter.old_spte)) {  in clear_dirty_pt_masked()
1696 			if (is_writable_pte(iter.old_spte))  in clear_dirty_pt_masked()
1697 				new_spte = iter.old_spte & ~PT_WRITABLE_MASK;  in clear_dirty_pt_masked()
1701 			if (iter.old_spte & shadow_dirty_mask)  in clear_dirty_pt_masked()
1702 				new_spte = iter.old_spte & ~shadow_dirty_mask;  in clear_dirty_pt_masked()
1749 		    !is_shadow_present_pte(iter.old_spte))  in zap_collapsible_spte_range()
1757 		if (is_last_spte(iter.old_spte, iter.level))  in zap_collapsible_spte_range()
1815 		if (!is_shadow_present_pte(iter.old_spte) ||  in write_protect_gfn()
1816 		    !is_last_spte(iter.old_spte, iter.level))  in write_protect_gfn()
1819 		new_spte = iter.old_spte &  in write_protect_gfn()
1822 		if (new_spte == iter.old_spte)  in write_protect_gfn()
1871 		sptes[leaf] = iter.old_spte;  in kvm_tdp_mmu_get_walk()
1897 		*spte = iter.old_spte;  in kvm_tdp_mmu_fast_pf_get_last_sptep()