1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_PSI_TYPES_H 3 #define _LINUX_PSI_TYPES_H 4 5 #include <linux/kthread.h> 6 #include <linux/seqlock.h> 7 #include <linux/types.h> 8 #include <linux/kref.h> 9 #include <linux/wait.h> 10 11 #ifdef CONFIG_PSI 12 13 /* Tracked task states */ 14 enum psi_task_count { 15 NR_IOWAIT, 16 NR_MEMSTALL, 17 NR_RUNNING, 18 /* 19 * For IO and CPU stalls the presence of running/oncpu tasks 20 * in the domain means a partial rather than a full stall. 21 * For memory it's not so simple because of page reclaimers: 22 * they are running/oncpu while representing a stall. To tell 23 * whether a domain has productivity left or not, we need to 24 * distinguish between regular running (i.e. productive) 25 * threads and memstall ones. 26 */ 27 NR_MEMSTALL_RUNNING, 28 NR_PSI_TASK_COUNTS = 4, 29 }; 30 31 /* Task state bitmasks */ 32 #define TSK_IOWAIT (1 << NR_IOWAIT) 33 #define TSK_MEMSTALL (1 << NR_MEMSTALL) 34 #define TSK_RUNNING (1 << NR_RUNNING) 35 #define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING) 36 37 /* Only one task can be scheduled, no corresponding task count */ 38 #define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS) 39 40 /* Resources that workloads could be stalled on */ 41 enum psi_res { 42 PSI_IO, 43 PSI_MEM, 44 PSI_CPU, 45 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 46 PSI_IRQ, 47 #endif 48 NR_PSI_RESOURCES, 49 }; 50 51 /* 52 * Pressure states for each resource: 53 * 54 * SOME: Stalled tasks & working tasks 55 * FULL: Stalled tasks & no working tasks 56 */ 57 enum psi_states { 58 PSI_IO_SOME, 59 PSI_IO_FULL, 60 PSI_MEM_SOME, 61 PSI_MEM_FULL, 62 PSI_CPU_SOME, 63 PSI_CPU_FULL, 64 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 65 PSI_IRQ_FULL, 66 #endif 67 /* Only per-CPU, to weigh the CPU in the global average: */ 68 PSI_NONIDLE, 69 NR_PSI_STATES, 70 }; 71 72 /* Use one bit in the state mask to track TSK_ONCPU */ 73 #define PSI_ONCPU (1 << NR_PSI_STATES) 74 75 /* Flag whether to re-arm avgs_work, see details in get_recent_times() */ 76 #define PSI_STATE_RESCHEDULE (1 << (NR_PSI_STATES + 1)) 77 78 enum psi_aggregators { 79 PSI_AVGS = 0, 80 PSI_POLL, 81 NR_PSI_AGGREGATORS, 82 }; 83 84 struct psi_group_cpu { 85 /* 1st cacheline updated by the scheduler */ 86 87 /* Aggregator needs to know of concurrent changes */ 88 seqcount_t seq ____cacheline_aligned_in_smp; 89 90 /* States of the tasks belonging to this group */ 91 unsigned int tasks[NR_PSI_TASK_COUNTS]; 92 93 /* Aggregate pressure state derived from the tasks */ 94 u32 state_mask; 95 96 /* Period time sampling buckets for each state of interest (ns) */ 97 u32 times[NR_PSI_STATES]; 98 99 /* Time of last task change in this group (rq_clock) */ 100 u64 state_start; 101 102 /* 2nd cacheline updated by the aggregator */ 103 104 /* Delta detection against the sampling buckets */ 105 u32 times_prev[NR_PSI_AGGREGATORS][NR_PSI_STATES] 106 ____cacheline_aligned_in_smp; 107 }; 108 109 /* PSI growth tracking window */ 110 struct psi_window { 111 /* Window size in ns */ 112 u64 size; 113 114 /* Start time of the current window in ns */ 115 u64 start_time; 116 117 /* Value at the start of the window */ 118 u64 start_value; 119 120 /* Value growth in the previous window */ 121 u64 prev_growth; 122 }; 123 124 struct psi_trigger { 125 /* PSI state being monitored by the trigger */ 126 enum psi_states state; 127 128 /* User-spacified threshold in ns */ 129 u64 threshold; 130 131 /* List node inside triggers list */ 132 struct list_head node; 133 134 /* Backpointer needed during trigger destruction */ 135 struct psi_group *group; 136 137 /* Wait queue for polling */ 138 wait_queue_head_t event_wait; 139 140 /* Kernfs file for cgroup triggers */ 141 struct kernfs_open_file *of; 142 143 /* Pending event flag */ 144 int event; 145 146 /* Tracking window */ 147 struct psi_window win; 148 149 /* 150 * Time last event was generated. Used for rate-limiting 151 * events to one per window 152 */ 153 u64 last_event_time; 154 155 /* Deferred event(s) from previous ratelimit window */ 156 bool pending_event; 157 158 /* Trigger type - PSI_AVGS for unprivileged, PSI_POLL for RT */ 159 enum psi_aggregators aggregator; 160 }; 161 162 struct psi_group { 163 struct psi_group *parent; 164 bool enabled; 165 166 /* Protects data used by the aggregator */ 167 struct mutex avgs_lock; 168 169 /* Per-cpu task state & time tracking */ 170 struct psi_group_cpu __percpu *pcpu; 171 172 /* Running pressure averages */ 173 u64 avg_total[NR_PSI_STATES - 1]; 174 u64 avg_last_update; 175 u64 avg_next_update; 176 177 /* Aggregator work control */ 178 struct delayed_work avgs_work; 179 180 /* Unprivileged triggers against N*PSI_FREQ windows */ 181 struct list_head avg_triggers; 182 u32 avg_nr_triggers[NR_PSI_STATES - 1]; 183 184 /* Total stall times and sampled pressure averages */ 185 u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1]; 186 unsigned long avg[NR_PSI_STATES - 1][3]; 187 188 /* Monitor RT polling work control */ 189 struct task_struct __rcu *rtpoll_task; 190 struct timer_list rtpoll_timer; 191 wait_queue_head_t rtpoll_wait; 192 atomic_t rtpoll_wakeup; 193 atomic_t rtpoll_scheduled; 194 195 /* Protects data used by the monitor */ 196 struct mutex rtpoll_trigger_lock; 197 198 /* Configured RT polling triggers */ 199 struct list_head rtpoll_triggers; 200 u32 rtpoll_nr_triggers[NR_PSI_STATES - 1]; 201 u32 rtpoll_states; 202 u64 rtpoll_min_period; 203 204 /* Total stall times at the start of RT polling monitor activation */ 205 u64 rtpoll_total[NR_PSI_STATES - 1]; 206 u64 rtpoll_next_update; 207 u64 rtpoll_until; 208 }; 209 210 #else /* CONFIG_PSI */ 211 212 #define NR_PSI_RESOURCES 0 213 214 struct psi_group { }; 215 216 #endif /* CONFIG_PSI */ 217 218 #endif /* _LINUX_PSI_TYPES_H */ 219