Lines Matching refs:gmap
34 static struct gmap *gmap_alloc(unsigned long limit) in gmap_alloc()
36 struct gmap *gmap; in gmap_alloc() local
58 gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL_ACCOUNT); in gmap_alloc()
59 if (!gmap) in gmap_alloc()
61 INIT_LIST_HEAD(&gmap->crst_list); in gmap_alloc()
62 INIT_LIST_HEAD(&gmap->children); in gmap_alloc()
63 INIT_LIST_HEAD(&gmap->pt_list); in gmap_alloc()
64 INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL_ACCOUNT); in gmap_alloc()
65 INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC | __GFP_ACCOUNT); in gmap_alloc()
66 INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC | __GFP_ACCOUNT); in gmap_alloc()
67 spin_lock_init(&gmap->guest_table_lock); in gmap_alloc()
68 spin_lock_init(&gmap->shadow_lock); in gmap_alloc()
69 refcount_set(&gmap->ref_count, 1); in gmap_alloc()
74 list_add(&page->lru, &gmap->crst_list); in gmap_alloc()
77 gmap->table = table; in gmap_alloc()
78 gmap->asce = atype | _ASCE_TABLE_LENGTH | in gmap_alloc()
80 gmap->asce_end = limit; in gmap_alloc()
81 return gmap; in gmap_alloc()
84 kfree(gmap); in gmap_alloc()
96 struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit) in gmap_create()
98 struct gmap *gmap; in gmap_create() local
101 gmap = gmap_alloc(limit); in gmap_create()
102 if (!gmap) in gmap_create()
104 gmap->mm = mm; in gmap_create()
106 list_add_rcu(&gmap->list, &mm->context.gmap_list); in gmap_create()
108 gmap_asce = gmap->asce; in gmap_create()
113 return gmap; in gmap_create()
117 static void gmap_flush_tlb(struct gmap *gmap) in gmap_flush_tlb() argument
120 __tlb_flush_idte(gmap->asce); in gmap_flush_tlb()
182 static void gmap_free(struct gmap *gmap) in gmap_free() argument
187 if (!(gmap_is_shadow(gmap) && gmap->removed)) in gmap_free()
188 gmap_flush_tlb(gmap); in gmap_free()
190 list_for_each_entry_safe(page, next, &gmap->crst_list, lru) in gmap_free()
192 gmap_radix_tree_free(&gmap->guest_to_host); in gmap_free()
193 gmap_radix_tree_free(&gmap->host_to_guest); in gmap_free()
196 if (gmap_is_shadow(gmap)) { in gmap_free()
198 list_for_each_entry_safe(page, next, &gmap->pt_list, lru) in gmap_free()
200 gmap_rmap_radix_tree_free(&gmap->host_to_rmap); in gmap_free()
202 gmap_put(gmap->parent); in gmap_free()
205 kfree(gmap); in gmap_free()
214 struct gmap *gmap_get(struct gmap *gmap) in gmap_get() argument
216 refcount_inc(&gmap->ref_count); in gmap_get()
217 return gmap; in gmap_get()
227 void gmap_put(struct gmap *gmap) in gmap_put() argument
229 if (refcount_dec_and_test(&gmap->ref_count)) in gmap_put()
230 gmap_free(gmap); in gmap_put()
238 void gmap_remove(struct gmap *gmap) in gmap_remove() argument
240 struct gmap *sg, *next; in gmap_remove()
244 if (!list_empty(&gmap->children)) { in gmap_remove()
245 spin_lock(&gmap->shadow_lock); in gmap_remove()
246 list_for_each_entry_safe(sg, next, &gmap->children, list) { in gmap_remove()
250 spin_unlock(&gmap->shadow_lock); in gmap_remove()
253 spin_lock(&gmap->mm->context.lock); in gmap_remove()
254 list_del_rcu(&gmap->list); in gmap_remove()
255 if (list_empty(&gmap->mm->context.gmap_list)) in gmap_remove()
257 else if (list_is_singular(&gmap->mm->context.gmap_list)) in gmap_remove()
258 gmap_asce = list_first_entry(&gmap->mm->context.gmap_list, in gmap_remove()
259 struct gmap, list)->asce; in gmap_remove()
262 WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce); in gmap_remove()
263 spin_unlock(&gmap->mm->context.lock); in gmap_remove()
266 gmap_put(gmap); in gmap_remove()
274 void gmap_enable(struct gmap *gmap) in gmap_enable() argument
276 S390_lowcore.gmap = (unsigned long) gmap; in gmap_enable()
284 void gmap_disable(struct gmap *gmap) in gmap_disable() argument
286 S390_lowcore.gmap = 0UL; in gmap_disable()
295 struct gmap *gmap_get_enabled(void) in gmap_get_enabled()
297 return (struct gmap *) S390_lowcore.gmap; in gmap_get_enabled()
304 static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, in gmap_alloc_table() argument
316 spin_lock(&gmap->guest_table_lock); in gmap_alloc_table()
318 list_add(&page->lru, &gmap->crst_list); in gmap_alloc_table()
324 spin_unlock(&gmap->guest_table_lock); in gmap_alloc_table()
355 static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr) in __gmap_unlink_by_vmaddr() argument
360 BUG_ON(gmap_is_shadow(gmap)); in __gmap_unlink_by_vmaddr()
361 spin_lock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
362 entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT); in __gmap_unlink_by_vmaddr()
367 spin_unlock(&gmap->guest_table_lock); in __gmap_unlink_by_vmaddr()
378 static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr) in __gmap_unmap_by_gaddr() argument
382 vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host, in __gmap_unmap_by_gaddr()
384 return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0; in __gmap_unmap_by_gaddr()
395 int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len) in gmap_unmap_segment() argument
400 BUG_ON(gmap_is_shadow(gmap)); in gmap_unmap_segment()
407 mmap_write_lock(gmap->mm); in gmap_unmap_segment()
409 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_unmap_segment()
410 mmap_write_unlock(gmap->mm); in gmap_unmap_segment()
412 gmap_flush_tlb(gmap); in gmap_unmap_segment()
426 int gmap_map_segment(struct gmap *gmap, unsigned long from, in gmap_map_segment() argument
432 BUG_ON(gmap_is_shadow(gmap)); in gmap_map_segment()
436 from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end) in gmap_map_segment()
440 mmap_write_lock(gmap->mm); in gmap_map_segment()
443 flush |= __gmap_unmap_by_gaddr(gmap, to + off); in gmap_map_segment()
445 if (radix_tree_insert(&gmap->guest_to_host, in gmap_map_segment()
450 mmap_write_unlock(gmap->mm); in gmap_map_segment()
452 gmap_flush_tlb(gmap); in gmap_map_segment()
455 gmap_unmap_segment(gmap, to, len); in gmap_map_segment()
473 unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr) in __gmap_translate() argument
478 radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT); in __gmap_translate()
493 unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr) in gmap_translate() argument
497 mmap_read_lock(gmap->mm); in gmap_translate()
498 rc = __gmap_translate(gmap, gaddr); in gmap_translate()
499 mmap_read_unlock(gmap->mm); in gmap_translate()
513 struct gmap *gmap; in gmap_unlink() local
517 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_unlink()
518 flush = __gmap_unlink_by_vmaddr(gmap, vmaddr); in gmap_unlink()
520 gmap_flush_tlb(gmap); in gmap_unlink()
525 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new,
539 int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) in __gmap_link() argument
551 BUG_ON(gmap_is_shadow(gmap)); in __gmap_link()
553 table = gmap->table; in __gmap_link()
554 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { in __gmap_link()
557 gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, in __gmap_link()
562 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { in __gmap_link()
565 gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, in __gmap_link()
570 if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { in __gmap_link()
573 gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, in __gmap_link()
580 mm = gmap->mm; in __gmap_link()
593 if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m) in __gmap_link()
600 spin_lock(&gmap->guest_table_lock); in __gmap_link()
602 rc = radix_tree_insert(&gmap->host_to_guest, in __gmap_link()
618 gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr); in __gmap_link()
620 spin_unlock(&gmap->guest_table_lock); in __gmap_link()
635 int gmap_fault(struct gmap *gmap, unsigned long gaddr, in gmap_fault() argument
642 mmap_read_lock(gmap->mm); in gmap_fault()
646 vmaddr = __gmap_translate(gmap, gaddr); in gmap_fault()
651 if (fixup_user_fault(gmap->mm, vmaddr, fault_flags, in gmap_fault()
663 rc = __gmap_link(gmap, gaddr, vmaddr); in gmap_fault()
665 mmap_read_unlock(gmap->mm); in gmap_fault()
673 void __gmap_zap(struct gmap *gmap, unsigned long gaddr) in __gmap_zap() argument
681 vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host, in __gmap_zap()
686 vma = vma_lookup(gmap->mm, vmaddr); in __gmap_zap()
691 ptep = get_locked_pte(gmap->mm, vmaddr, &ptl); in __gmap_zap()
693 ptep_zap_unused(gmap->mm, vmaddr, ptep, 0); in __gmap_zap()
700 void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) in gmap_discard() argument
705 mmap_read_lock(gmap->mm); in gmap_discard()
710 radix_tree_lookup(&gmap->guest_to_host, in gmap_discard()
716 vma = find_vma(gmap->mm, vmaddr); in gmap_discard()
728 mmap_read_unlock(gmap->mm); in gmap_discard()
766 static void gmap_call_notifier(struct gmap *gmap, unsigned long start, in gmap_call_notifier() argument
772 nb->notifier_call(gmap, start, end); in gmap_call_notifier()
793 static inline unsigned long *gmap_table_walk(struct gmap *gmap, in gmap_table_walk() argument
796 const int asce_type = gmap->asce & _ASCE_TYPE_MASK; in gmap_table_walk()
797 unsigned long *table = gmap->table; in gmap_table_walk()
799 if (gmap_is_shadow(gmap) && gmap->removed) in gmap_table_walk()
855 static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_walk() argument
860 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_walk()
862 table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */ in gmap_pte_op_walk()
865 return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl); in gmap_pte_op_walk()
879 static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, in gmap_pte_op_fixup() argument
882 struct mm_struct *mm = gmap->mm; in gmap_pte_op_fixup()
886 BUG_ON(gmap_is_shadow(gmap)); in gmap_pte_op_fixup()
894 return __gmap_link(gmap, gaddr, vmaddr); in gmap_pte_op_fixup()
915 static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr) in gmap_pmd_op_walk() argument
919 BUG_ON(gmap_is_shadow(gmap)); in gmap_pmd_op_walk()
920 pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1); in gmap_pmd_op_walk()
925 if (!gmap->mm->context.allow_gmap_hpage_1m) in gmap_pmd_op_walk()
928 spin_lock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
930 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
936 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_walk()
945 static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp) in gmap_pmd_op_end() argument
948 spin_unlock(&gmap->guest_table_lock); in gmap_pmd_op_end()
965 static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pmd() argument
978 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
984 gmap_pmdp_xchg(gmap, pmdp, new, gaddr); in gmap_protect_pmd()
1010 static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr, in gmap_protect_pte() argument
1021 ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl); in gmap_protect_pte()
1028 rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits); in gmap_protect_pte()
1046 static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr, in gmap_protect_range() argument
1053 BUG_ON(gmap_is_shadow(gmap)); in gmap_protect_range()
1056 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_protect_range()
1059 rc = gmap_protect_pte(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1066 rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot, in gmap_protect_range()
1074 gmap_pmd_op_end(gmap, pmdp); in gmap_protect_range()
1081 vmaddr = __gmap_translate(gmap, gaddr); in gmap_protect_range()
1084 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot); in gmap_protect_range()
1106 int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, in gmap_mprotect_notify() argument
1111 if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap)) in gmap_mprotect_notify()
1115 mmap_read_lock(gmap->mm); in gmap_mprotect_notify()
1116 rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT); in gmap_mprotect_notify()
1117 mmap_read_unlock(gmap->mm); in gmap_mprotect_notify()
1135 int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val) in gmap_read_table() argument
1142 if (gmap_is_shadow(gmap)) in gmap_read_table()
1147 ptep = gmap_pte_op_walk(gmap, gaddr, &ptl); in gmap_read_table()
1162 vmaddr = __gmap_translate(gmap, gaddr); in gmap_read_table()
1167 rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ); in gmap_read_table()
1183 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr, in gmap_insert_rmap()
1218 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr, in gmap_protect_rmap()
1221 struct gmap *parent; in gmap_protect_rmap()
1299 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) in gmap_unshadow_page()
1319 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_pgt()
1336 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_pgt()
1365 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_sgt()
1393 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) in gmap_unshadow_sgt()
1422 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r3t()
1450 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r3t()
1479 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r2t()
1507 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) in gmap_unshadow_r2t()
1536 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, in __gmap_unshadow_r1t()
1566 static void gmap_unshadow(struct gmap *sg) in gmap_unshadow()
1603 static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce, in gmap_find_shadow()
1606 struct gmap *sg; in gmap_find_shadow()
1632 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level) in gmap_shadow_valid()
1655 struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, in gmap_shadow()
1658 struct gmap *sg, *new; in gmap_shadow()
1747 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, in gmap_shadow_r2t()
1831 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, in gmap_shadow_r3t()
1915 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, in gmap_shadow_sgt()
1999 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr, in gmap_shadow_pgt_lookup()
2039 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, in gmap_shadow_pgt()
2118 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte) in gmap_shadow_page()
2120 struct gmap *parent; in gmap_shadow_page()
2186 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, in gmap_shadow_notify()
2253 struct gmap *gmap, *sg, *next; in ptep_notify() local
2258 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in ptep_notify()
2259 spin_lock(&gmap->guest_table_lock); in ptep_notify()
2260 table = radix_tree_lookup(&gmap->host_to_guest, in ptep_notify()
2264 spin_unlock(&gmap->guest_table_lock); in ptep_notify()
2268 if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) { in ptep_notify()
2269 spin_lock(&gmap->shadow_lock); in ptep_notify()
2271 &gmap->children, list) in ptep_notify()
2273 spin_unlock(&gmap->shadow_lock); in ptep_notify()
2276 gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1); in ptep_notify()
2282 static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp, in pmdp_notify_gmap() argument
2286 gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1); in pmdp_notify_gmap()
2299 static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new, in gmap_pmdp_xchg() argument
2303 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_xchg()
2306 __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce, in gmap_pmdp_xchg()
2319 struct gmap *gmap; in gmap_pmdp_clear() local
2323 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_clear()
2324 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2325 pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_clear()
2329 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_clear()
2336 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_clear()
2372 struct gmap *gmap; in gmap_pmdp_idte_local() local
2376 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_local()
2377 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2378 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_local()
2383 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_local()
2388 gmap->asce, IDTE_LOCAL); in gmap_pmdp_idte_local()
2393 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_local()
2407 struct gmap *gmap; in gmap_pmdp_idte_global() local
2411 list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { in gmap_pmdp_idte_global()
2412 spin_lock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2413 entry = radix_tree_delete(&gmap->host_to_guest, in gmap_pmdp_idte_global()
2418 pmdp_notify_gmap(gmap, pmdp, gaddr); in gmap_pmdp_idte_global()
2423 gmap->asce, IDTE_GLOBAL); in gmap_pmdp_idte_global()
2430 spin_unlock(&gmap->guest_table_lock); in gmap_pmdp_idte_global()
2445 static bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp, in gmap_test_and_clear_dirty_pmd() argument
2458 gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0); in gmap_test_and_clear_dirty_pmd()
2472 void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4], in gmap_sync_dirty_log_pmd() argument
2480 pmdp = gmap_pmd_op_walk(gmap, gaddr); in gmap_sync_dirty_log_pmd()
2485 if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr)) in gmap_sync_dirty_log_pmd()
2489 ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl); in gmap_sync_dirty_log_pmd()
2492 if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep)) in gmap_sync_dirty_log_pmd()
2497 gmap_pmd_op_end(gmap, pmdp); in gmap_sync_dirty_log_pmd()
2799 void s390_unlist_old_asce(struct gmap *gmap) in s390_unlist_old_asce() argument
2803 old = virt_to_page(gmap->table); in s390_unlist_old_asce()
2804 spin_lock(&gmap->guest_table_lock); in s390_unlist_old_asce()
2821 spin_unlock(&gmap->guest_table_lock); in s390_unlist_old_asce()
2835 int s390_replace_asce(struct gmap *gmap) in s390_replace_asce() argument
2841 s390_unlist_old_asce(gmap); in s390_replace_asce()
2847 memcpy(table, gmap->table, 1UL << (CRST_ALLOC_ORDER + PAGE_SHIFT)); in s390_replace_asce()
2854 spin_lock(&gmap->guest_table_lock); in s390_replace_asce()
2855 list_add(&page->lru, &gmap->crst_list); in s390_replace_asce()
2856 spin_unlock(&gmap->guest_table_lock); in s390_replace_asce()
2859 asce = (gmap->asce & ~_ASCE_ORIGIN) | __pa(table); in s390_replace_asce()
2860 WRITE_ONCE(gmap->asce, asce); in s390_replace_asce()
2861 WRITE_ONCE(gmap->mm->context.gmap_asce, asce); in s390_replace_asce()
2862 WRITE_ONCE(gmap->table, table); in s390_replace_asce()