1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020 ARM Ltd.
4 */
5 #ifndef __ASM_MTE_H
6 #define __ASM_MTE_H
7
8 #include <asm/compiler.h>
9 #include <asm/mte-def.h>
10
11 #ifndef __ASSEMBLY__
12
13 #include <linux/bitfield.h>
14 #include <linux/kasan-enabled.h>
15 #include <linux/page-flags.h>
16 #include <linux/sched.h>
17 #include <linux/types.h>
18
19 #include <asm/pgtable-types.h>
20
21 void mte_clear_page_tags(void *addr);
22 unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
23 unsigned long n);
24 unsigned long mte_copy_tags_to_user(void __user *to, void *from,
25 unsigned long n);
26 int mte_save_tags(struct page *page);
27 void mte_save_page_tags(const void *page_addr, void *tag_storage);
28 bool mte_restore_tags(swp_entry_t entry, struct page *page);
29 void mte_restore_page_tags(void *page_addr, const void *tag_storage);
30 void mte_invalidate_tags(int type, pgoff_t offset);
31 void mte_invalidate_tags_area(int type);
32 void *mte_allocate_tag_storage(void);
33 void mte_free_tag_storage(char *storage);
34
35 #ifdef CONFIG_ARM64_MTE
36
37 /* track which pages have valid allocation tags */
38 #define PG_mte_tagged PG_arch_2
39
40 void mte_zero_clear_page_tags(void *addr);
41 void mte_sync_tags(pte_t old_pte, pte_t pte);
42 void mte_copy_page_tags(void *kto, const void *kfrom);
43 void mte_thread_init_user(void);
44 void mte_thread_switch(struct task_struct *next);
45 void mte_cpu_setup(void);
46 void mte_suspend_enter(void);
47 void mte_suspend_exit(void);
48 long set_mte_ctrl(struct task_struct *task, unsigned long arg);
49 long get_mte_ctrl(struct task_struct *task);
50 int mte_ptrace_copy_tags(struct task_struct *child, long request,
51 unsigned long addr, unsigned long data);
52 size_t mte_probe_user_range(const char __user *uaddr, size_t size);
53
54 #else /* CONFIG_ARM64_MTE */
55
56 /* unused if !CONFIG_ARM64_MTE, silence the compiler */
57 #define PG_mte_tagged 0
58
mte_zero_clear_page_tags(void * addr)59 static inline void mte_zero_clear_page_tags(void *addr)
60 {
61 }
mte_sync_tags(pte_t old_pte,pte_t pte)62 static inline void mte_sync_tags(pte_t old_pte, pte_t pte)
63 {
64 }
mte_copy_page_tags(void * kto,const void * kfrom)65 static inline void mte_copy_page_tags(void *kto, const void *kfrom)
66 {
67 }
mte_thread_init_user(void)68 static inline void mte_thread_init_user(void)
69 {
70 }
mte_thread_switch(struct task_struct * next)71 static inline void mte_thread_switch(struct task_struct *next)
72 {
73 }
mte_suspend_enter(void)74 static inline void mte_suspend_enter(void)
75 {
76 }
mte_suspend_exit(void)77 static inline void mte_suspend_exit(void)
78 {
79 }
set_mte_ctrl(struct task_struct * task,unsigned long arg)80 static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
81 {
82 return 0;
83 }
get_mte_ctrl(struct task_struct * task)84 static inline long get_mte_ctrl(struct task_struct *task)
85 {
86 return 0;
87 }
mte_ptrace_copy_tags(struct task_struct * child,long request,unsigned long addr,unsigned long data)88 static inline int mte_ptrace_copy_tags(struct task_struct *child,
89 long request, unsigned long addr,
90 unsigned long data)
91 {
92 return -EIO;
93 }
94
95 #endif /* CONFIG_ARM64_MTE */
96
mte_disable_tco_entry(struct task_struct * task)97 static inline void mte_disable_tco_entry(struct task_struct *task)
98 {
99 if (!system_supports_mte())
100 return;
101
102 /*
103 * Re-enable tag checking (TCO set on exception entry). This is only
104 * necessary if MTE is enabled in either the kernel or the userspace
105 * task in synchronous or asymmetric mode (SCTLR_EL1.TCF0 bit 0 is set
106 * for both). With MTE disabled in the kernel and disabled or
107 * asynchronous in userspace, tag check faults (including in uaccesses)
108 * are not reported, therefore there is no need to re-enable checking.
109 * This is beneficial on microarchitectures where re-enabling TCO is
110 * expensive.
111 */
112 if (kasan_hw_tags_enabled() ||
113 (task->thread.sctlr_user & (1UL << SCTLR_EL1_TCF0_SHIFT)))
114 asm volatile(SET_PSTATE_TCO(0));
115 }
116
117 #ifdef CONFIG_KASAN_HW_TAGS
118 /* Whether the MTE asynchronous mode is enabled. */
119 DECLARE_STATIC_KEY_FALSE(mte_async_or_asymm_mode);
120
system_uses_mte_async_or_asymm_mode(void)121 static inline bool system_uses_mte_async_or_asymm_mode(void)
122 {
123 return static_branch_unlikely(&mte_async_or_asymm_mode);
124 }
125
126 void mte_check_tfsr_el1(void);
127
mte_check_tfsr_entry(void)128 static inline void mte_check_tfsr_entry(void)
129 {
130 if (!system_supports_mte())
131 return;
132
133 mte_check_tfsr_el1();
134 }
135
mte_check_tfsr_exit(void)136 static inline void mte_check_tfsr_exit(void)
137 {
138 if (!system_supports_mte())
139 return;
140
141 /*
142 * The asynchronous faults are sync'ed automatically with
143 * TFSR_EL1 on kernel entry but for exit an explicit dsb()
144 * is required.
145 */
146 dsb(nsh);
147 isb();
148
149 mte_check_tfsr_el1();
150 }
151 #else
system_uses_mte_async_or_asymm_mode(void)152 static inline bool system_uses_mte_async_or_asymm_mode(void)
153 {
154 return false;
155 }
mte_check_tfsr_el1(void)156 static inline void mte_check_tfsr_el1(void)
157 {
158 }
mte_check_tfsr_entry(void)159 static inline void mte_check_tfsr_entry(void)
160 {
161 }
mte_check_tfsr_exit(void)162 static inline void mte_check_tfsr_exit(void)
163 {
164 }
165 #endif /* CONFIG_KASAN_HW_TAGS */
166
167 #endif /* __ASSEMBLY__ */
168 #endif /* __ASM_MTE_H */
169