1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * This driver adds support for HNS3 PMU iEP device. Related perf events are
4 * bandwidth, latency, packet rate, interrupt rate etc.
5 *
6 * Copyright (C) 2022 HiSilicon Limited
7 */
8 #include <linux/bitfield.h>
9 #include <linux/bitmap.h>
10 #include <linux/bug.h>
11 #include <linux/cpuhotplug.h>
12 #include <linux/cpumask.h>
13 #include <linux/delay.h>
14 #include <linux/device.h>
15 #include <linux/err.h>
16 #include <linux/interrupt.h>
17 #include <linux/iopoll.h>
18 #include <linux/io-64-nonatomic-hi-lo.h>
19 #include <linux/irq.h>
20 #include <linux/kernel.h>
21 #include <linux/list.h>
22 #include <linux/module.h>
23 #include <linux/pci.h>
24 #include <linux/pci-epf.h>
25 #include <linux/perf_event.h>
26 #include <linux/smp.h>
27
28 /* registers offset address */
29 #define HNS3_PMU_REG_GLOBAL_CTRL 0x0000
30 #define HNS3_PMU_REG_CLOCK_FREQ 0x0020
31 #define HNS3_PMU_REG_BDF 0x0fe0
32 #define HNS3_PMU_REG_VERSION 0x0fe4
33 #define HNS3_PMU_REG_DEVICE_ID 0x0fe8
34
35 #define HNS3_PMU_REG_EVENT_OFFSET 0x1000
36 #define HNS3_PMU_REG_EVENT_SIZE 0x1000
37 #define HNS3_PMU_REG_EVENT_CTRL_LOW 0x00
38 #define HNS3_PMU_REG_EVENT_CTRL_HIGH 0x04
39 #define HNS3_PMU_REG_EVENT_INTR_STATUS 0x08
40 #define HNS3_PMU_REG_EVENT_INTR_MASK 0x0c
41 #define HNS3_PMU_REG_EVENT_COUNTER 0x10
42 #define HNS3_PMU_REG_EVENT_EXT_COUNTER 0x18
43 #define HNS3_PMU_REG_EVENT_QID_CTRL 0x28
44 #define HNS3_PMU_REG_EVENT_QID_PARA 0x2c
45
46 #define HNS3_PMU_FILTER_SUPPORT_GLOBAL BIT(0)
47 #define HNS3_PMU_FILTER_SUPPORT_PORT BIT(1)
48 #define HNS3_PMU_FILTER_SUPPORT_PORT_TC BIT(2)
49 #define HNS3_PMU_FILTER_SUPPORT_FUNC BIT(3)
50 #define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE BIT(4)
51 #define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR BIT(5)
52
53 #define HNS3_PMU_FILTER_ALL_TC 0xf
54 #define HNS3_PMU_FILTER_ALL_QUEUE 0xffff
55
56 #define HNS3_PMU_CTRL_SUBEVENT_S 4
57 #define HNS3_PMU_CTRL_FILTER_MODE_S 24
58
59 #define HNS3_PMU_GLOBAL_START BIT(0)
60
61 #define HNS3_PMU_EVENT_STATUS_RESET BIT(11)
62 #define HNS3_PMU_EVENT_EN BIT(12)
63 #define HNS3_PMU_EVENT_OVERFLOW_RESTART BIT(15)
64
65 #define HNS3_PMU_QID_PARA_FUNC_S 0
66 #define HNS3_PMU_QID_PARA_QUEUE_S 16
67
68 #define HNS3_PMU_QID_CTRL_REQ_ENABLE BIT(0)
69 #define HNS3_PMU_QID_CTRL_DONE BIT(1)
70 #define HNS3_PMU_QID_CTRL_MISS BIT(2)
71
72 #define HNS3_PMU_INTR_MASK_OVERFLOW BIT(1)
73
74 #define HNS3_PMU_MAX_HW_EVENTS 8
75
76 /*
77 * Each hardware event contains two registers (counter and ext_counter) for
78 * bandwidth, packet rate, latency and interrupt rate. These two registers will
79 * be triggered to run at the same when a hardware event is enabled. The meaning
80 * of counter and ext_counter of different event type are different, their
81 * meaning show as follow:
82 *
83 * +----------------+------------------+---------------+
84 * | event type | counter | ext_counter |
85 * +----------------+------------------+---------------+
86 * | bandwidth | byte number | cycle number |
87 * +----------------+------------------+---------------+
88 * | packet rate | packet number | cycle number |
89 * +----------------+------------------+---------------+
90 * | latency | cycle number | packet number |
91 * +----------------+------------------+---------------+
92 * | interrupt rate | interrupt number | cycle number |
93 * +----------------+------------------+---------------+
94 *
95 * The cycle number indicates increment of counter of hardware timer, the
96 * frequency of hardware timer can be read from hw_clk_freq file.
97 *
98 * Performance of each hardware event is calculated by: counter / ext_counter.
99 *
100 * Since processing of data is preferred to be done in userspace, we expose
101 * ext_counter as a separate event for userspace and use bit 16 to indicate it.
102 * For example, event 0x00001 and 0x10001 are actually one event for hardware
103 * because bit 0-15 are same. If the bit 16 of one event is 0 means to read
104 * counter register, otherwise means to read ext_counter register.
105 */
106 /* bandwidth events */
107 #define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM 0x00001
108 #define HNS3_PMU_EVT_BW_SSU_EGU_TIME 0x10001
109 #define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM 0x00002
110 #define HNS3_PMU_EVT_BW_SSU_RPU_TIME 0x10002
111 #define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM 0x00003
112 #define HNS3_PMU_EVT_BW_SSU_ROCE_TIME 0x10003
113 #define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM 0x00004
114 #define HNS3_PMU_EVT_BW_ROCE_SSU_TIME 0x10004
115 #define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM 0x00005
116 #define HNS3_PMU_EVT_BW_TPU_SSU_TIME 0x10005
117 #define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM 0x00006
118 #define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME 0x10006
119 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM 0x00008
120 #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME 0x10008
121 #define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM 0x00009
122 #define HNS3_PMU_EVT_BW_WR_FBD_TIME 0x10009
123 #define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM 0x0000a
124 #define HNS3_PMU_EVT_BW_WR_EBD_TIME 0x1000a
125 #define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM 0x0000b
126 #define HNS3_PMU_EVT_BW_RD_FBD_TIME 0x1000b
127 #define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM 0x0000c
128 #define HNS3_PMU_EVT_BW_RD_EBD_TIME 0x1000c
129 #define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM 0x0000d
130 #define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME 0x1000d
131 #define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM 0x0000e
132 #define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME 0x1000e
133 #define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM 0x0000f
134 #define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME 0x1000f
135 #define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM 0x00010
136 #define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME 0x10010
137
138 /* packet rate events */
139 #define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM 0x00100
140 #define HNS3_PMU_EVT_PPS_IGU_SSU_TIME 0x10100
141 #define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM 0x00101
142 #define HNS3_PMU_EVT_PPS_SSU_EGU_TIME 0x10101
143 #define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM 0x00102
144 #define HNS3_PMU_EVT_PPS_SSU_RPU_TIME 0x10102
145 #define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM 0x00103
146 #define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME 0x10103
147 #define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM 0x00104
148 #define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME 0x10104
149 #define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM 0x00105
150 #define HNS3_PMU_EVT_PPS_TPU_SSU_TIME 0x10105
151 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM 0x00106
152 #define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME 0x10106
153 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM 0x00107
154 #define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME 0x10107
155 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM 0x00108
156 #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME 0x10108
157 #define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM 0x00109
158 #define HNS3_PMU_EVT_PPS_WR_FBD_TIME 0x10109
159 #define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM 0x0010a
160 #define HNS3_PMU_EVT_PPS_WR_EBD_TIME 0x1010a
161 #define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM 0x0010b
162 #define HNS3_PMU_EVT_PPS_RD_FBD_TIME 0x1010b
163 #define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM 0x0010c
164 #define HNS3_PMU_EVT_PPS_RD_EBD_TIME 0x1010c
165 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM 0x0010d
166 #define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME 0x1010d
167 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM 0x0010e
168 #define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME 0x1010e
169 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM 0x0010f
170 #define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME 0x1010f
171 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM 0x00110
172 #define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME 0x10110
173 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM 0x00111
174 #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME 0x10111
175 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM 0x00112
176 #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME 0x10112
177
178 /* latency events */
179 #define HNS3_PMU_EVT_DLY_TX_PUSH_TIME 0x00202
180 #define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM 0x10202
181 #define HNS3_PMU_EVT_DLY_TX_TIME 0x00204
182 #define HNS3_PMU_EVT_DLY_TX_PACKET_NUM 0x10204
183 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME 0x00206
184 #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM 0x10206
185 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME 0x00207
186 #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM 0x10207
187 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME 0x00208
188 #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM 0x10208
189 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME 0x00209
190 #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM 0x10209
191 #define HNS3_PMU_EVT_DLY_RPU_TIME 0x0020e
192 #define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM 0x1020e
193 #define HNS3_PMU_EVT_DLY_TPU_TIME 0x0020f
194 #define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM 0x1020f
195 #define HNS3_PMU_EVT_DLY_RPE_TIME 0x00210
196 #define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM 0x10210
197 #define HNS3_PMU_EVT_DLY_TPE_TIME 0x00211
198 #define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM 0x10211
199 #define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME 0x00212
200 #define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM 0x10212
201 #define HNS3_PMU_EVT_DLY_WR_FBD_TIME 0x00213
202 #define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM 0x10213
203 #define HNS3_PMU_EVT_DLY_WR_EBD_TIME 0x00214
204 #define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM 0x10214
205 #define HNS3_PMU_EVT_DLY_RD_FBD_TIME 0x00215
206 #define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM 0x10215
207 #define HNS3_PMU_EVT_DLY_RD_EBD_TIME 0x00216
208 #define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM 0x10216
209 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME 0x00217
210 #define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM 0x10217
211 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME 0x00218
212 #define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM 0x10218
213 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME 0x00219
214 #define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM 0x10219
215 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME 0x0021a
216 #define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM 0x1021a
217 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME 0x0021c
218 #define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM 0x1021c
219
220 /* interrupt rate events */
221 #define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM 0x00300
222 #define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME 0x10300
223
224 /* filter mode supported by each bandwidth event */
225 #define HNS3_PMU_FILTER_BW_SSU_EGU 0x07
226 #define HNS3_PMU_FILTER_BW_SSU_RPU 0x1f
227 #define HNS3_PMU_FILTER_BW_SSU_ROCE 0x0f
228 #define HNS3_PMU_FILTER_BW_ROCE_SSU 0x0f
229 #define HNS3_PMU_FILTER_BW_TPU_SSU 0x1f
230 #define HNS3_PMU_FILTER_BW_RPU_RCBRX 0x11
231 #define HNS3_PMU_FILTER_BW_RCBTX_TXSCH 0x11
232 #define HNS3_PMU_FILTER_BW_WR_FBD 0x1b
233 #define HNS3_PMU_FILTER_BW_WR_EBD 0x11
234 #define HNS3_PMU_FILTER_BW_RD_FBD 0x01
235 #define HNS3_PMU_FILTER_BW_RD_EBD 0x1b
236 #define HNS3_PMU_FILTER_BW_RD_PAY_M0 0x01
237 #define HNS3_PMU_FILTER_BW_RD_PAY_M1 0x01
238 #define HNS3_PMU_FILTER_BW_WR_PAY_M0 0x01
239 #define HNS3_PMU_FILTER_BW_WR_PAY_M1 0x01
240
241 /* filter mode supported by each packet rate event */
242 #define HNS3_PMU_FILTER_PPS_IGU_SSU 0x07
243 #define HNS3_PMU_FILTER_PPS_SSU_EGU 0x07
244 #define HNS3_PMU_FILTER_PPS_SSU_RPU 0x1f
245 #define HNS3_PMU_FILTER_PPS_SSU_ROCE 0x0f
246 #define HNS3_PMU_FILTER_PPS_ROCE_SSU 0x0f
247 #define HNS3_PMU_FILTER_PPS_TPU_SSU 0x1f
248 #define HNS3_PMU_FILTER_PPS_RPU_RCBRX 0x11
249 #define HNS3_PMU_FILTER_PPS_RCBTX_TPU 0x1f
250 #define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH 0x11
251 #define HNS3_PMU_FILTER_PPS_WR_FBD 0x1b
252 #define HNS3_PMU_FILTER_PPS_WR_EBD 0x11
253 #define HNS3_PMU_FILTER_PPS_RD_FBD 0x01
254 #define HNS3_PMU_FILTER_PPS_RD_EBD 0x1b
255 #define HNS3_PMU_FILTER_PPS_RD_PAY_M0 0x01
256 #define HNS3_PMU_FILTER_PPS_RD_PAY_M1 0x01
257 #define HNS3_PMU_FILTER_PPS_WR_PAY_M0 0x01
258 #define HNS3_PMU_FILTER_PPS_WR_PAY_M1 0x01
259 #define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE 0x01
260 #define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE 0x01
261
262 /* filter mode supported by each latency event */
263 #define HNS3_PMU_FILTER_DLY_TX_PUSH 0x01
264 #define HNS3_PMU_FILTER_DLY_TX 0x01
265 #define HNS3_PMU_FILTER_DLY_SSU_TX_NIC 0x07
266 #define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE 0x07
267 #define HNS3_PMU_FILTER_DLY_SSU_RX_NIC 0x07
268 #define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE 0x07
269 #define HNS3_PMU_FILTER_DLY_RPU 0x11
270 #define HNS3_PMU_FILTER_DLY_TPU 0x1f
271 #define HNS3_PMU_FILTER_DLY_RPE 0x01
272 #define HNS3_PMU_FILTER_DLY_TPE 0x0b
273 #define HNS3_PMU_FILTER_DLY_TPE_PUSH 0x1b
274 #define HNS3_PMU_FILTER_DLY_WR_FBD 0x1b
275 #define HNS3_PMU_FILTER_DLY_WR_EBD 0x11
276 #define HNS3_PMU_FILTER_DLY_RD_FBD 0x01
277 #define HNS3_PMU_FILTER_DLY_RD_EBD 0x1b
278 #define HNS3_PMU_FILTER_DLY_RD_PAY_M0 0x01
279 #define HNS3_PMU_FILTER_DLY_RD_PAY_M1 0x01
280 #define HNS3_PMU_FILTER_DLY_WR_PAY_M0 0x01
281 #define HNS3_PMU_FILTER_DLY_WR_PAY_M1 0x01
282 #define HNS3_PMU_FILTER_DLY_MSIX_WRITE 0x01
283
284 /* filter mode supported by each interrupt rate event */
285 #define HNS3_PMU_FILTER_INTR_MSIX_NIC 0x01
286
287 enum hns3_pmu_hw_filter_mode {
288 HNS3_PMU_HW_FILTER_GLOBAL,
289 HNS3_PMU_HW_FILTER_PORT,
290 HNS3_PMU_HW_FILTER_PORT_TC,
291 HNS3_PMU_HW_FILTER_FUNC,
292 HNS3_PMU_HW_FILTER_FUNC_QUEUE,
293 HNS3_PMU_HW_FILTER_FUNC_INTR,
294 };
295
296 struct hns3_pmu_event_attr {
297 u32 event;
298 u16 filter_support;
299 };
300
301 struct hns3_pmu {
302 struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS];
303 struct hlist_node node;
304 struct pci_dev *pdev;
305 struct pmu pmu;
306 void __iomem *base;
307 int irq;
308 int on_cpu;
309 u32 identifier;
310 u32 hw_clk_freq; /* hardware clock frequency of PMU */
311 /* maximum and minimum bdf allowed by PMU */
312 u16 bdf_min;
313 u16 bdf_max;
314 };
315
316 #define to_hns3_pmu(p) (container_of((p), struct hns3_pmu, pmu))
317
318 #define GET_PCI_DEVFN(bdf) ((bdf) & 0xff)
319
320 #define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff)
321 #define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07))
322 #define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func))
323
324 #define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end) \
325 static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \
326 { \
327 return FIELD_GET(GENMASK_ULL(_end, _start), \
328 event->attr._config); \
329 }
330
331 HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7);
332 HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15);
333 HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16);
334 HNS3_PMU_FILTER_ATTR(port, config1, 0, 3);
335 HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7);
336 HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23);
337 HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39);
338 HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51);
339 HNS3_PMU_FILTER_ATTR(global, config1, 52, 52);
340
341 #define HNS3_BW_EVT_BYTE_NUM(_name) (&(struct hns3_pmu_event_attr) {\
342 HNS3_PMU_EVT_BW_##_name##_BYTE_NUM, \
343 HNS3_PMU_FILTER_BW_##_name})
344 #define HNS3_BW_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
345 HNS3_PMU_EVT_BW_##_name##_TIME, \
346 HNS3_PMU_FILTER_BW_##_name})
347 #define HNS3_PPS_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\
348 HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM, \
349 HNS3_PMU_FILTER_PPS_##_name})
350 #define HNS3_PPS_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
351 HNS3_PMU_EVT_PPS_##_name##_TIME, \
352 HNS3_PMU_FILTER_PPS_##_name})
353 #define HNS3_DLY_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
354 HNS3_PMU_EVT_DLY_##_name##_TIME, \
355 HNS3_PMU_FILTER_DLY_##_name})
356 #define HNS3_DLY_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\
357 HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM, \
358 HNS3_PMU_FILTER_DLY_##_name})
359 #define HNS3_INTR_EVT_INTR_NUM(_name) (&(struct hns3_pmu_event_attr) {\
360 HNS3_PMU_EVT_PPS_##_name##_INTR_NUM, \
361 HNS3_PMU_FILTER_INTR_##_name})
362 #define HNS3_INTR_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
363 HNS3_PMU_EVT_PPS_##_name##_TIME, \
364 HNS3_PMU_FILTER_INTR_##_name})
365
hns3_pmu_format_show(struct device * dev,struct device_attribute * attr,char * buf)366 static ssize_t hns3_pmu_format_show(struct device *dev,
367 struct device_attribute *attr, char *buf)
368 {
369 struct dev_ext_attribute *eattr;
370
371 eattr = container_of(attr, struct dev_ext_attribute, attr);
372
373 return sysfs_emit(buf, "%s\n", (char *)eattr->var);
374 }
375
hns3_pmu_event_show(struct device * dev,struct device_attribute * attr,char * buf)376 static ssize_t hns3_pmu_event_show(struct device *dev,
377 struct device_attribute *attr, char *buf)
378 {
379 struct hns3_pmu_event_attr *event;
380 struct dev_ext_attribute *eattr;
381
382 eattr = container_of(attr, struct dev_ext_attribute, attr);
383 event = eattr->var;
384
385 return sysfs_emit(buf, "config=0x%x\n", event->event);
386 }
387
hns3_pmu_filter_mode_show(struct device * dev,struct device_attribute * attr,char * buf)388 static ssize_t hns3_pmu_filter_mode_show(struct device *dev,
389 struct device_attribute *attr,
390 char *buf)
391 {
392 struct hns3_pmu_event_attr *event;
393 struct dev_ext_attribute *eattr;
394 int len;
395
396 eattr = container_of(attr, struct dev_ext_attribute, attr);
397 event = eattr->var;
398
399 len = sysfs_emit_at(buf, 0, "filter mode supported: ");
400 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)
401 len += sysfs_emit_at(buf, len, "global ");
402 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)
403 len += sysfs_emit_at(buf, len, "port ");
404 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)
405 len += sysfs_emit_at(buf, len, "port-tc ");
406 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)
407 len += sysfs_emit_at(buf, len, "func ");
408 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)
409 len += sysfs_emit_at(buf, len, "func-queue ");
410 if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)
411 len += sysfs_emit_at(buf, len, "func-intr ");
412
413 len += sysfs_emit_at(buf, len, "\n");
414
415 return len;
416 }
417
418 #define HNS3_PMU_ATTR(_name, _func, _config) \
419 (&((struct dev_ext_attribute[]) { \
420 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \
421 })[0].attr.attr)
422
423 #define HNS3_PMU_FORMAT_ATTR(_name, _format) \
424 HNS3_PMU_ATTR(_name, hns3_pmu_format_show, (void *)_format)
425 #define HNS3_PMU_EVENT_ATTR(_name, _event) \
426 HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event)
427 #define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \
428 HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event)
429
430 #define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \
431 HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
432 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
433 #define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \
434 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
435 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
436 #define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \
437 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
438 HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
439 #define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \
440 HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
441 HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
442
443 #define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \
444 HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
445 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
446 #define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \
447 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
448 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
449 #define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \
450 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
451 HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
452 #define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \
453 HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
454 HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
455
456 static u8 hns3_pmu_hw_filter_modes[] = {
457 HNS3_PMU_HW_FILTER_GLOBAL,
458 HNS3_PMU_HW_FILTER_PORT,
459 HNS3_PMU_HW_FILTER_PORT_TC,
460 HNS3_PMU_HW_FILTER_FUNC,
461 HNS3_PMU_HW_FILTER_FUNC_QUEUE,
462 HNS3_PMU_HW_FILTER_FUNC_INTR,
463 };
464
465 #define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \
466 ((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)])
467
identifier_show(struct device * dev,struct device_attribute * attr,char * buf)468 static ssize_t identifier_show(struct device *dev,
469 struct device_attribute *attr, char *buf)
470 {
471 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
472
473 return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier);
474 }
475 static DEVICE_ATTR_RO(identifier);
476
cpumask_show(struct device * dev,struct device_attribute * attr,char * buf)477 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
478 char *buf)
479 {
480 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
481
482 return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu);
483 }
484 static DEVICE_ATTR_RO(cpumask);
485
bdf_min_show(struct device * dev,struct device_attribute * attr,char * buf)486 static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr,
487 char *buf)
488 {
489 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
490 u16 bdf = hns3_pmu->bdf_min;
491
492 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
493 PCI_SLOT(bdf), PCI_FUNC(bdf));
494 }
495 static DEVICE_ATTR_RO(bdf_min);
496
bdf_max_show(struct device * dev,struct device_attribute * attr,char * buf)497 static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr,
498 char *buf)
499 {
500 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
501 u16 bdf = hns3_pmu->bdf_max;
502
503 return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
504 PCI_SLOT(bdf), PCI_FUNC(bdf));
505 }
506 static DEVICE_ATTR_RO(bdf_max);
507
hw_clk_freq_show(struct device * dev,struct device_attribute * attr,char * buf)508 static ssize_t hw_clk_freq_show(struct device *dev,
509 struct device_attribute *attr, char *buf)
510 {
511 struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
512
513 return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq);
514 }
515 static DEVICE_ATTR_RO(hw_clk_freq);
516
517 static struct attribute *hns3_pmu_events_attr[] = {
518 /* bandwidth events */
519 HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU),
520 HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU),
521 HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE),
522 HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU),
523 HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU),
524 HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
525 HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
526 HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD),
527 HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD),
528 HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD),
529 HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD),
530 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0),
531 HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1),
532 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0),
533 HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1),
534
535 /* packet rate events */
536 HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU),
537 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU),
538 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU),
539 HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE),
540 HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU),
541 HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU),
542 HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
543 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
544 HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
545 HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD),
546 HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD),
547 HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD),
548 HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD),
549 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0),
550 HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1),
551 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0),
552 HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1),
553 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
554 HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
555
556 /* latency events */
557 HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH),
558 HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX),
559 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
560 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
561 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
562 HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
563 HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU),
564 HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU),
565 HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE),
566 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE),
567 HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH),
568 HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD),
569 HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD),
570 HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD),
571 HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD),
572 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0),
573 HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1),
574 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0),
575 HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1),
576 HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE),
577
578 /* interrupt rate events */
579 HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC),
580
581 NULL
582 };
583
584 static struct attribute *hns3_pmu_filter_mode_attr[] = {
585 /* bandwidth events */
586 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU),
587 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU),
588 HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE),
589 HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU),
590 HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU),
591 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
592 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
593 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD),
594 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD),
595 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD),
596 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD),
597 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0),
598 HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1),
599 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0),
600 HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1),
601
602 /* packet rate events */
603 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU),
604 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU),
605 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU),
606 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE),
607 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU),
608 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU),
609 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
610 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
611 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
612 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD),
613 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD),
614 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD),
615 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD),
616 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0),
617 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1),
618 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0),
619 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1),
620 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
621 HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
622
623 /* latency events */
624 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH),
625 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX),
626 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
627 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
628 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
629 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
630 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU),
631 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU),
632 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE),
633 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE),
634 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH),
635 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD),
636 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD),
637 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD),
638 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD),
639 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0),
640 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1),
641 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0),
642 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1),
643 HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE),
644
645 /* interrupt rate events */
646 HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC),
647
648 NULL
649 };
650
651 static struct attribute_group hns3_pmu_events_group = {
652 .name = "events",
653 .attrs = hns3_pmu_events_attr,
654 };
655
656 static struct attribute_group hns3_pmu_filter_mode_group = {
657 .name = "filtermode",
658 .attrs = hns3_pmu_filter_mode_attr,
659 };
660
661 static struct attribute *hns3_pmu_format_attr[] = {
662 HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"),
663 HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"),
664 HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"),
665 HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"),
666 HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"),
667 HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"),
668 HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"),
669 HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"),
670 HNS3_PMU_FORMAT_ATTR(global, "config1:52"),
671 NULL
672 };
673
674 static struct attribute_group hns3_pmu_format_group = {
675 .name = "format",
676 .attrs = hns3_pmu_format_attr,
677 };
678
679 static struct attribute *hns3_pmu_cpumask_attrs[] = {
680 &dev_attr_cpumask.attr,
681 NULL
682 };
683
684 static struct attribute_group hns3_pmu_cpumask_attr_group = {
685 .attrs = hns3_pmu_cpumask_attrs,
686 };
687
688 static struct attribute *hns3_pmu_identifier_attrs[] = {
689 &dev_attr_identifier.attr,
690 NULL
691 };
692
693 static struct attribute_group hns3_pmu_identifier_attr_group = {
694 .attrs = hns3_pmu_identifier_attrs,
695 };
696
697 static struct attribute *hns3_pmu_bdf_range_attrs[] = {
698 &dev_attr_bdf_min.attr,
699 &dev_attr_bdf_max.attr,
700 NULL
701 };
702
703 static struct attribute_group hns3_pmu_bdf_range_attr_group = {
704 .attrs = hns3_pmu_bdf_range_attrs,
705 };
706
707 static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = {
708 &dev_attr_hw_clk_freq.attr,
709 NULL
710 };
711
712 static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = {
713 .attrs = hns3_pmu_hw_clk_freq_attrs,
714 };
715
716 static const struct attribute_group *hns3_pmu_attr_groups[] = {
717 &hns3_pmu_events_group,
718 &hns3_pmu_filter_mode_group,
719 &hns3_pmu_format_group,
720 &hns3_pmu_cpumask_attr_group,
721 &hns3_pmu_identifier_attr_group,
722 &hns3_pmu_bdf_range_attr_group,
723 &hns3_pmu_hw_clk_freq_attr_group,
724 NULL
725 };
726
hns3_pmu_get_event(struct perf_event * event)727 static u32 hns3_pmu_get_event(struct perf_event *event)
728 {
729 return hns3_pmu_get_ext_counter_used(event) << 16 |
730 hns3_pmu_get_event_type(event) << 8 |
731 hns3_pmu_get_subevent(event);
732 }
733
hns3_pmu_get_real_event(struct perf_event * event)734 static u32 hns3_pmu_get_real_event(struct perf_event *event)
735 {
736 return hns3_pmu_get_event_type(event) << 8 |
737 hns3_pmu_get_subevent(event);
738 }
739
hns3_pmu_get_offset(u32 offset,u32 idx)740 static u32 hns3_pmu_get_offset(u32 offset, u32 idx)
741 {
742 return offset + HNS3_PMU_REG_EVENT_OFFSET +
743 HNS3_PMU_REG_EVENT_SIZE * idx;
744 }
745
hns3_pmu_readl(struct hns3_pmu * hns3_pmu,u32 reg_offset,u32 idx)746 static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
747 {
748 u32 offset = hns3_pmu_get_offset(reg_offset, idx);
749
750 return readl(hns3_pmu->base + offset);
751 }
752
hns3_pmu_writel(struct hns3_pmu * hns3_pmu,u32 reg_offset,u32 idx,u32 val)753 static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
754 u32 val)
755 {
756 u32 offset = hns3_pmu_get_offset(reg_offset, idx);
757
758 writel(val, hns3_pmu->base + offset);
759 }
760
hns3_pmu_readq(struct hns3_pmu * hns3_pmu,u32 reg_offset,u32 idx)761 static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
762 {
763 u32 offset = hns3_pmu_get_offset(reg_offset, idx);
764
765 return readq(hns3_pmu->base + offset);
766 }
767
hns3_pmu_writeq(struct hns3_pmu * hns3_pmu,u32 reg_offset,u32 idx,u64 val)768 static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
769 u64 val)
770 {
771 u32 offset = hns3_pmu_get_offset(reg_offset, idx);
772
773 writeq(val, hns3_pmu->base + offset);
774 }
775
hns3_pmu_cmp_event(struct perf_event * target,struct perf_event * event)776 static bool hns3_pmu_cmp_event(struct perf_event *target,
777 struct perf_event *event)
778 {
779 return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event);
780 }
781
hns3_pmu_find_related_event_idx(struct hns3_pmu * hns3_pmu,struct perf_event * event)782 static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu,
783 struct perf_event *event)
784 {
785 struct perf_event *sibling;
786 int hw_event_used = 0;
787 int idx;
788
789 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
790 sibling = hns3_pmu->hw_events[idx];
791 if (!sibling)
792 continue;
793
794 hw_event_used++;
795
796 if (!hns3_pmu_cmp_event(sibling, event))
797 continue;
798
799 /* Related events is used in group */
800 if (sibling->group_leader == event->group_leader)
801 return idx;
802 }
803
804 /* No related event and all hardware events are used up */
805 if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS)
806 return -EBUSY;
807
808 /* No related event and there is extra hardware events can be use */
809 return -ENOENT;
810 }
811
hns3_pmu_get_event_idx(struct hns3_pmu * hns3_pmu)812 static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu)
813 {
814 int idx;
815
816 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
817 if (!hns3_pmu->hw_events[idx])
818 return idx;
819 }
820
821 return -EBUSY;
822 }
823
hns3_pmu_valid_bdf(struct hns3_pmu * hns3_pmu,u16 bdf)824 static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf)
825 {
826 struct pci_dev *pdev;
827
828 if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) {
829 pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf);
830 return false;
831 }
832
833 pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus),
834 PCI_BUS_NUM(bdf),
835 GET_PCI_DEVFN(bdf));
836 if (!pdev) {
837 pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf);
838 return false;
839 }
840
841 pci_dev_put(pdev);
842 return true;
843 }
844
hns3_pmu_set_qid_para(struct hns3_pmu * hns3_pmu,u32 idx,u16 bdf,u16 queue)845 static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
846 u16 queue)
847 {
848 u32 val;
849
850 val = GET_PCI_DEVFN(bdf);
851 val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S;
852 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val);
853 }
854
hns3_pmu_qid_req_start(struct hns3_pmu * hns3_pmu,u32 idx)855 static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx)
856 {
857 bool queue_id_valid = false;
858 u32 reg_qid_ctrl, val;
859 int err;
860
861 /* enable queue id request */
862 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx,
863 HNS3_PMU_QID_CTRL_REQ_ENABLE);
864
865 reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx);
866 err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val,
867 val & HNS3_PMU_QID_CTRL_DONE, 1, 1000);
868 if (err == -ETIMEDOUT) {
869 pci_err(hns3_pmu->pdev, "QID request timeout!\n");
870 goto out;
871 }
872
873 queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS);
874
875 out:
876 /* disable qid request and clear status */
877 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0);
878
879 return queue_id_valid;
880 }
881
hns3_pmu_valid_queue(struct hns3_pmu * hns3_pmu,u32 idx,u16 bdf,u16 queue)882 static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
883 u16 queue)
884 {
885 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue);
886
887 return hns3_pmu_qid_req_start(hns3_pmu, idx);
888 }
889
hns3_pmu_get_pmu_event(u32 event)890 static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event)
891 {
892 struct hns3_pmu_event_attr *pmu_event;
893 struct dev_ext_attribute *eattr;
894 struct device_attribute *dattr;
895 struct attribute *attr;
896 u32 i;
897
898 for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) {
899 attr = hns3_pmu_events_attr[i];
900 dattr = container_of(attr, struct device_attribute, attr);
901 eattr = container_of(dattr, struct dev_ext_attribute, attr);
902 pmu_event = eattr->var;
903
904 if (event == pmu_event->event)
905 return pmu_event;
906 }
907
908 return NULL;
909 }
910
hns3_pmu_set_func_mode(struct perf_event * event,struct hns3_pmu * hns3_pmu)911 static int hns3_pmu_set_func_mode(struct perf_event *event,
912 struct hns3_pmu *hns3_pmu)
913 {
914 struct hw_perf_event *hwc = &event->hw;
915 u16 bdf = hns3_pmu_get_bdf(event);
916
917 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
918 return -ENOENT;
919
920 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC);
921
922 return 0;
923 }
924
hns3_pmu_set_func_queue_mode(struct perf_event * event,struct hns3_pmu * hns3_pmu)925 static int hns3_pmu_set_func_queue_mode(struct perf_event *event,
926 struct hns3_pmu *hns3_pmu)
927 {
928 u16 queue_id = hns3_pmu_get_queue(event);
929 struct hw_perf_event *hwc = &event->hw;
930 u16 bdf = hns3_pmu_get_bdf(event);
931
932 if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
933 return -ENOENT;
934
935 if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) {
936 pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id);
937 return -ENOENT;
938 }
939
940 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE);
941
942 return 0;
943 }
944
945 static bool
hns3_pmu_is_enabled_global_mode(struct perf_event * event,struct hns3_pmu_event_attr * pmu_event)946 hns3_pmu_is_enabled_global_mode(struct perf_event *event,
947 struct hns3_pmu_event_attr *pmu_event)
948 {
949 u8 global = hns3_pmu_get_global(event);
950
951 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL))
952 return false;
953
954 return global;
955 }
956
hns3_pmu_is_enabled_func_mode(struct perf_event * event,struct hns3_pmu_event_attr * pmu_event)957 static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event,
958 struct hns3_pmu_event_attr *pmu_event)
959 {
960 u16 queue_id = hns3_pmu_get_queue(event);
961 u16 bdf = hns3_pmu_get_bdf(event);
962
963 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC))
964 return false;
965 else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE)
966 return false;
967
968 return bdf;
969 }
970
971 static bool
hns3_pmu_is_enabled_func_queue_mode(struct perf_event * event,struct hns3_pmu_event_attr * pmu_event)972 hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event,
973 struct hns3_pmu_event_attr *pmu_event)
974 {
975 u16 queue_id = hns3_pmu_get_queue(event);
976 u16 bdf = hns3_pmu_get_bdf(event);
977
978 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE))
979 return false;
980 else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE)
981 return false;
982
983 return bdf;
984 }
985
hns3_pmu_is_enabled_port_mode(struct perf_event * event,struct hns3_pmu_event_attr * pmu_event)986 static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event,
987 struct hns3_pmu_event_attr *pmu_event)
988 {
989 u8 tc_id = hns3_pmu_get_tc(event);
990
991 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT))
992 return false;
993
994 return tc_id == HNS3_PMU_FILTER_ALL_TC;
995 }
996
997 static bool
hns3_pmu_is_enabled_port_tc_mode(struct perf_event * event,struct hns3_pmu_event_attr * pmu_event)998 hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event,
999 struct hns3_pmu_event_attr *pmu_event)
1000 {
1001 u8 tc_id = hns3_pmu_get_tc(event);
1002
1003 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC))
1004 return false;
1005
1006 return tc_id != HNS3_PMU_FILTER_ALL_TC;
1007 }
1008
1009 static bool
hns3_pmu_is_enabled_func_intr_mode(struct perf_event * event,struct hns3_pmu * hns3_pmu,struct hns3_pmu_event_attr * pmu_event)1010 hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event,
1011 struct hns3_pmu *hns3_pmu,
1012 struct hns3_pmu_event_attr *pmu_event)
1013 {
1014 u16 bdf = hns3_pmu_get_bdf(event);
1015
1016 if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR))
1017 return false;
1018
1019 return hns3_pmu_valid_bdf(hns3_pmu, bdf);
1020 }
1021
hns3_pmu_select_filter_mode(struct perf_event * event,struct hns3_pmu * hns3_pmu)1022 static int hns3_pmu_select_filter_mode(struct perf_event *event,
1023 struct hns3_pmu *hns3_pmu)
1024 {
1025 u32 event_id = hns3_pmu_get_event(event);
1026 struct hw_perf_event *hwc = &event->hw;
1027 struct hns3_pmu_event_attr *pmu_event;
1028
1029 pmu_event = hns3_pmu_get_pmu_event(event_id);
1030 if (!pmu_event) {
1031 pci_err(hns3_pmu->pdev, "Invalid pmu event\n");
1032 return -ENOENT;
1033 }
1034
1035 if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) {
1036 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL);
1037 return 0;
1038 }
1039
1040 if (hns3_pmu_is_enabled_func_mode(event, pmu_event))
1041 return hns3_pmu_set_func_mode(event, hns3_pmu);
1042
1043 if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event))
1044 return hns3_pmu_set_func_queue_mode(event, hns3_pmu);
1045
1046 if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) {
1047 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT);
1048 return 0;
1049 }
1050
1051 if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) {
1052 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC);
1053 return 0;
1054 }
1055
1056 if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) {
1057 HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR);
1058 return 0;
1059 }
1060
1061 return -ENOENT;
1062 }
1063
hns3_pmu_validate_event_group(struct perf_event * event)1064 static bool hns3_pmu_validate_event_group(struct perf_event *event)
1065 {
1066 struct perf_event *sibling, *leader = event->group_leader;
1067 struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS];
1068 int counters = 1;
1069 int num;
1070
1071 event_group[0] = leader;
1072 if (!is_software_event(leader)) {
1073 if (leader->pmu != event->pmu)
1074 return false;
1075
1076 if (leader != event && !hns3_pmu_cmp_event(leader, event))
1077 event_group[counters++] = event;
1078 }
1079
1080 for_each_sibling_event(sibling, event->group_leader) {
1081 if (is_software_event(sibling))
1082 continue;
1083
1084 if (sibling->pmu != event->pmu)
1085 return false;
1086
1087 for (num = 0; num < counters; num++) {
1088 if (hns3_pmu_cmp_event(event_group[num], sibling))
1089 break;
1090 }
1091
1092 if (num == counters)
1093 event_group[counters++] = sibling;
1094 }
1095
1096 return counters <= HNS3_PMU_MAX_HW_EVENTS;
1097 }
1098
hns3_pmu_get_filter_condition(struct perf_event * event)1099 static u32 hns3_pmu_get_filter_condition(struct perf_event *event)
1100 {
1101 struct hw_perf_event *hwc = &event->hw;
1102 u16 intr_id = hns3_pmu_get_intr(event);
1103 u8 port_id = hns3_pmu_get_port(event);
1104 u16 bdf = hns3_pmu_get_bdf(event);
1105 u8 tc_id = hns3_pmu_get_tc(event);
1106 u8 filter_mode;
1107
1108 filter_mode = *(u8 *)hwc->addr_filters;
1109 switch (filter_mode) {
1110 case HNS3_PMU_HW_FILTER_PORT:
1111 return FILTER_CONDITION_PORT(port_id);
1112 case HNS3_PMU_HW_FILTER_PORT_TC:
1113 return FILTER_CONDITION_PORT_TC(port_id, tc_id);
1114 case HNS3_PMU_HW_FILTER_FUNC:
1115 case HNS3_PMU_HW_FILTER_FUNC_QUEUE:
1116 return GET_PCI_DEVFN(bdf);
1117 case HNS3_PMU_HW_FILTER_FUNC_INTR:
1118 return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id);
1119 default:
1120 break;
1121 }
1122
1123 return 0;
1124 }
1125
hns3_pmu_config_filter(struct perf_event * event)1126 static void hns3_pmu_config_filter(struct perf_event *event)
1127 {
1128 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1129 u8 event_type = hns3_pmu_get_event_type(event);
1130 u8 subevent_id = hns3_pmu_get_subevent(event);
1131 u16 queue_id = hns3_pmu_get_queue(event);
1132 struct hw_perf_event *hwc = &event->hw;
1133 u8 filter_mode = *(u8 *)hwc->addr_filters;
1134 u16 bdf = hns3_pmu_get_bdf(event);
1135 u32 idx = hwc->idx;
1136 u32 val;
1137
1138 val = event_type;
1139 val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S;
1140 val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S;
1141 val |= HNS3_PMU_EVENT_OVERFLOW_RESTART;
1142 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1143
1144 val = hns3_pmu_get_filter_condition(event);
1145 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val);
1146
1147 if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE)
1148 hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id);
1149 }
1150
hns3_pmu_enable_counter(struct hns3_pmu * hns3_pmu,struct hw_perf_event * hwc)1151 static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu,
1152 struct hw_perf_event *hwc)
1153 {
1154 u32 idx = hwc->idx;
1155 u32 val;
1156
1157 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1158 val |= HNS3_PMU_EVENT_EN;
1159 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1160 }
1161
hns3_pmu_disable_counter(struct hns3_pmu * hns3_pmu,struct hw_perf_event * hwc)1162 static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu,
1163 struct hw_perf_event *hwc)
1164 {
1165 u32 idx = hwc->idx;
1166 u32 val;
1167
1168 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1169 val &= ~HNS3_PMU_EVENT_EN;
1170 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1171 }
1172
hns3_pmu_enable_intr(struct hns3_pmu * hns3_pmu,struct hw_perf_event * hwc)1173 static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu,
1174 struct hw_perf_event *hwc)
1175 {
1176 u32 idx = hwc->idx;
1177 u32 val;
1178
1179 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
1180 val &= ~HNS3_PMU_INTR_MASK_OVERFLOW;
1181 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
1182 }
1183
hns3_pmu_disable_intr(struct hns3_pmu * hns3_pmu,struct hw_perf_event * hwc)1184 static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu,
1185 struct hw_perf_event *hwc)
1186 {
1187 u32 idx = hwc->idx;
1188 u32 val;
1189
1190 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
1191 val |= HNS3_PMU_INTR_MASK_OVERFLOW;
1192 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
1193 }
1194
hns3_pmu_clear_intr_status(struct hns3_pmu * hns3_pmu,u32 idx)1195 static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx)
1196 {
1197 u32 val;
1198
1199 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1200 val |= HNS3_PMU_EVENT_STATUS_RESET;
1201 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1202
1203 val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
1204 val &= ~HNS3_PMU_EVENT_STATUS_RESET;
1205 hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
1206 }
1207
hns3_pmu_read_counter(struct perf_event * event)1208 static u64 hns3_pmu_read_counter(struct perf_event *event)
1209 {
1210 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1211
1212 return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx);
1213 }
1214
hns3_pmu_write_counter(struct perf_event * event,u64 value)1215 static void hns3_pmu_write_counter(struct perf_event *event, u64 value)
1216 {
1217 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1218 u32 idx = event->hw.idx;
1219
1220 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value);
1221 hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value);
1222 }
1223
hns3_pmu_init_counter(struct perf_event * event)1224 static void hns3_pmu_init_counter(struct perf_event *event)
1225 {
1226 struct hw_perf_event *hwc = &event->hw;
1227
1228 local64_set(&hwc->prev_count, 0);
1229 hns3_pmu_write_counter(event, 0);
1230 }
1231
hns3_pmu_event_init(struct perf_event * event)1232 static int hns3_pmu_event_init(struct perf_event *event)
1233 {
1234 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1235 struct hw_perf_event *hwc = &event->hw;
1236 int idx;
1237 int ret;
1238
1239 if (event->attr.type != event->pmu->type)
1240 return -ENOENT;
1241
1242 /* Sampling is not supported */
1243 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1244 return -EOPNOTSUPP;
1245
1246 event->cpu = hns3_pmu->on_cpu;
1247
1248 idx = hns3_pmu_get_event_idx(hns3_pmu);
1249 if (idx < 0) {
1250 pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n",
1251 HNS3_PMU_MAX_HW_EVENTS);
1252 return -EBUSY;
1253 }
1254
1255 hwc->idx = idx;
1256
1257 ret = hns3_pmu_select_filter_mode(event, hns3_pmu);
1258 if (ret) {
1259 pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret);
1260 return ret;
1261 }
1262
1263 if (!hns3_pmu_validate_event_group(event)) {
1264 pci_err(hns3_pmu->pdev, "Invalid event group.\n");
1265 return -EINVAL;
1266 }
1267
1268 if (hns3_pmu_get_ext_counter_used(event))
1269 hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER;
1270 else
1271 hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER;
1272
1273 return 0;
1274 }
1275
hns3_pmu_read(struct perf_event * event)1276 static void hns3_pmu_read(struct perf_event *event)
1277 {
1278 struct hw_perf_event *hwc = &event->hw;
1279 u64 new_cnt, prev_cnt, delta;
1280
1281 do {
1282 prev_cnt = local64_read(&hwc->prev_count);
1283 new_cnt = hns3_pmu_read_counter(event);
1284 } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) !=
1285 prev_cnt);
1286
1287 delta = new_cnt - prev_cnt;
1288 local64_add(delta, &event->count);
1289 }
1290
hns3_pmu_start(struct perf_event * event,int flags)1291 static void hns3_pmu_start(struct perf_event *event, int flags)
1292 {
1293 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1294 struct hw_perf_event *hwc = &event->hw;
1295
1296 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
1297 return;
1298
1299 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1300 hwc->state = 0;
1301
1302 hns3_pmu_config_filter(event);
1303 hns3_pmu_init_counter(event);
1304 hns3_pmu_enable_intr(hns3_pmu, hwc);
1305 hns3_pmu_enable_counter(hns3_pmu, hwc);
1306
1307 perf_event_update_userpage(event);
1308 }
1309
hns3_pmu_stop(struct perf_event * event,int flags)1310 static void hns3_pmu_stop(struct perf_event *event, int flags)
1311 {
1312 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1313 struct hw_perf_event *hwc = &event->hw;
1314
1315 hns3_pmu_disable_counter(hns3_pmu, hwc);
1316 hns3_pmu_disable_intr(hns3_pmu, hwc);
1317
1318 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
1319 hwc->state |= PERF_HES_STOPPED;
1320
1321 if (hwc->state & PERF_HES_UPTODATE)
1322 return;
1323
1324 /* Read hardware counter and update the perf counter statistics */
1325 hns3_pmu_read(event);
1326 hwc->state |= PERF_HES_UPTODATE;
1327 }
1328
hns3_pmu_add(struct perf_event * event,int flags)1329 static int hns3_pmu_add(struct perf_event *event, int flags)
1330 {
1331 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1332 struct hw_perf_event *hwc = &event->hw;
1333 int idx;
1334
1335 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1336
1337 /* Check all working events to find a related event. */
1338 idx = hns3_pmu_find_related_event_idx(hns3_pmu, event);
1339 if (idx < 0 && idx != -ENOENT)
1340 return idx;
1341
1342 /* Current event shares an enabled hardware event with related event */
1343 if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) {
1344 hwc->idx = idx;
1345 goto start_count;
1346 }
1347
1348 idx = hns3_pmu_get_event_idx(hns3_pmu);
1349 if (idx < 0)
1350 return idx;
1351
1352 hwc->idx = idx;
1353 hns3_pmu->hw_events[idx] = event;
1354
1355 start_count:
1356 if (flags & PERF_EF_START)
1357 hns3_pmu_start(event, PERF_EF_RELOAD);
1358
1359 return 0;
1360 }
1361
hns3_pmu_del(struct perf_event * event,int flags)1362 static void hns3_pmu_del(struct perf_event *event, int flags)
1363 {
1364 struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
1365 struct hw_perf_event *hwc = &event->hw;
1366
1367 hns3_pmu_stop(event, PERF_EF_UPDATE);
1368 hns3_pmu->hw_events[hwc->idx] = NULL;
1369 perf_event_update_userpage(event);
1370 }
1371
hns3_pmu_enable(struct pmu * pmu)1372 static void hns3_pmu_enable(struct pmu *pmu)
1373 {
1374 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
1375 u32 val;
1376
1377 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1378 val |= HNS3_PMU_GLOBAL_START;
1379 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1380 }
1381
hns3_pmu_disable(struct pmu * pmu)1382 static void hns3_pmu_disable(struct pmu *pmu)
1383 {
1384 struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
1385 u32 val;
1386
1387 val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1388 val &= ~HNS3_PMU_GLOBAL_START;
1389 writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
1390 }
1391
hns3_pmu_alloc_pmu(struct pci_dev * pdev,struct hns3_pmu * hns3_pmu)1392 static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
1393 {
1394 u16 device_id;
1395 char *name;
1396 u32 val;
1397
1398 hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2];
1399 if (!hns3_pmu->base) {
1400 pci_err(pdev, "ioremap failed\n");
1401 return -ENOMEM;
1402 }
1403
1404 hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ);
1405
1406 val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF);
1407 hns3_pmu->bdf_min = val & 0xffff;
1408 hns3_pmu->bdf_max = val >> 16;
1409
1410 val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID);
1411 device_id = val & 0xffff;
1412 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id);
1413 if (!name)
1414 return -ENOMEM;
1415
1416 hns3_pmu->pdev = pdev;
1417 hns3_pmu->on_cpu = -1;
1418 hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION);
1419 hns3_pmu->pmu = (struct pmu) {
1420 .name = name,
1421 .module = THIS_MODULE,
1422 .event_init = hns3_pmu_event_init,
1423 .pmu_enable = hns3_pmu_enable,
1424 .pmu_disable = hns3_pmu_disable,
1425 .add = hns3_pmu_add,
1426 .del = hns3_pmu_del,
1427 .start = hns3_pmu_start,
1428 .stop = hns3_pmu_stop,
1429 .read = hns3_pmu_read,
1430 .task_ctx_nr = perf_invalid_context,
1431 .attr_groups = hns3_pmu_attr_groups,
1432 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1433 };
1434
1435 return 0;
1436 }
1437
hns3_pmu_irq(int irq,void * data)1438 static irqreturn_t hns3_pmu_irq(int irq, void *data)
1439 {
1440 struct hns3_pmu *hns3_pmu = data;
1441 u32 intr_status, idx;
1442
1443 for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
1444 intr_status = hns3_pmu_readl(hns3_pmu,
1445 HNS3_PMU_REG_EVENT_INTR_STATUS,
1446 idx);
1447
1448 /*
1449 * As each counter will restart from 0 when it is overflowed,
1450 * extra processing is no need, just clear interrupt status.
1451 */
1452 if (intr_status)
1453 hns3_pmu_clear_intr_status(hns3_pmu, idx);
1454 }
1455
1456 return IRQ_HANDLED;
1457 }
1458
hns3_pmu_online_cpu(unsigned int cpu,struct hlist_node * node)1459 static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
1460 {
1461 struct hns3_pmu *hns3_pmu;
1462
1463 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
1464 if (!hns3_pmu)
1465 return -ENODEV;
1466
1467 if (hns3_pmu->on_cpu == -1) {
1468 hns3_pmu->on_cpu = cpu;
1469 irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu));
1470 }
1471
1472 return 0;
1473 }
1474
hns3_pmu_offline_cpu(unsigned int cpu,struct hlist_node * node)1475 static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1476 {
1477 struct hns3_pmu *hns3_pmu;
1478 unsigned int target;
1479
1480 hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
1481 if (!hns3_pmu)
1482 return -ENODEV;
1483
1484 /* Nothing to do if this CPU doesn't own the PMU */
1485 if (hns3_pmu->on_cpu != cpu)
1486 return 0;
1487
1488 /* Choose a new CPU from all online cpus */
1489 target = cpumask_any_but(cpu_online_mask, cpu);
1490 if (target >= nr_cpu_ids)
1491 return 0;
1492
1493 perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target);
1494 hns3_pmu->on_cpu = target;
1495 irq_set_affinity(hns3_pmu->irq, cpumask_of(target));
1496
1497 return 0;
1498 }
1499
hns3_pmu_free_irq(void * data)1500 static void hns3_pmu_free_irq(void *data)
1501 {
1502 struct pci_dev *pdev = data;
1503
1504 pci_free_irq_vectors(pdev);
1505 }
1506
hns3_pmu_irq_register(struct pci_dev * pdev,struct hns3_pmu * hns3_pmu)1507 static int hns3_pmu_irq_register(struct pci_dev *pdev,
1508 struct hns3_pmu *hns3_pmu)
1509 {
1510 int irq, ret;
1511
1512 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
1513 if (ret < 0) {
1514 pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret);
1515 return ret;
1516 }
1517
1518 ret = devm_add_action(&pdev->dev, hns3_pmu_free_irq, pdev);
1519 if (ret) {
1520 pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret);
1521 return ret;
1522 }
1523
1524 irq = pci_irq_vector(pdev, 0);
1525 ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0,
1526 hns3_pmu->pmu.name, hns3_pmu);
1527 if (ret) {
1528 pci_err(pdev, "failed to register irq, ret = %d.\n", ret);
1529 return ret;
1530 }
1531
1532 hns3_pmu->irq = irq;
1533
1534 return 0;
1535 }
1536
hns3_pmu_init_pmu(struct pci_dev * pdev,struct hns3_pmu * hns3_pmu)1537 static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
1538 {
1539 int ret;
1540
1541 ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu);
1542 if (ret)
1543 return ret;
1544
1545 ret = hns3_pmu_irq_register(pdev, hns3_pmu);
1546 if (ret)
1547 return ret;
1548
1549 ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1550 &hns3_pmu->node);
1551 if (ret) {
1552 pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret);
1553 return ret;
1554 }
1555
1556 ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1);
1557 if (ret) {
1558 pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret);
1559 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1560 &hns3_pmu->node);
1561 }
1562
1563 return ret;
1564 }
1565
hns3_pmu_uninit_pmu(struct pci_dev * pdev)1566 static void hns3_pmu_uninit_pmu(struct pci_dev *pdev)
1567 {
1568 struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev);
1569
1570 perf_pmu_unregister(&hns3_pmu->pmu);
1571 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1572 &hns3_pmu->node);
1573 }
1574
hns3_pmu_init_dev(struct pci_dev * pdev)1575 static int hns3_pmu_init_dev(struct pci_dev *pdev)
1576 {
1577 int ret;
1578
1579 ret = pcim_enable_device(pdev);
1580 if (ret) {
1581 pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret);
1582 return ret;
1583 }
1584
1585 ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu");
1586 if (ret < 0) {
1587 pci_err(pdev, "failed to request pci region, ret = %d.\n", ret);
1588 return ret;
1589 }
1590
1591 pci_set_master(pdev);
1592
1593 return 0;
1594 }
1595
hns3_pmu_probe(struct pci_dev * pdev,const struct pci_device_id * id)1596 static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1597 {
1598 struct hns3_pmu *hns3_pmu;
1599 int ret;
1600
1601 hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL);
1602 if (!hns3_pmu)
1603 return -ENOMEM;
1604
1605 ret = hns3_pmu_init_dev(pdev);
1606 if (ret)
1607 return ret;
1608
1609 ret = hns3_pmu_init_pmu(pdev, hns3_pmu);
1610 if (ret) {
1611 pci_clear_master(pdev);
1612 return ret;
1613 }
1614
1615 pci_set_drvdata(pdev, hns3_pmu);
1616
1617 return ret;
1618 }
1619
hns3_pmu_remove(struct pci_dev * pdev)1620 static void hns3_pmu_remove(struct pci_dev *pdev)
1621 {
1622 hns3_pmu_uninit_pmu(pdev);
1623 pci_clear_master(pdev);
1624 pci_set_drvdata(pdev, NULL);
1625 }
1626
1627 static const struct pci_device_id hns3_pmu_ids[] = {
1628 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) },
1629 { 0, }
1630 };
1631 MODULE_DEVICE_TABLE(pci, hns3_pmu_ids);
1632
1633 static struct pci_driver hns3_pmu_driver = {
1634 .name = "hns3_pmu",
1635 .id_table = hns3_pmu_ids,
1636 .probe = hns3_pmu_probe,
1637 .remove = hns3_pmu_remove,
1638 };
1639
hns3_pmu_module_init(void)1640 static int __init hns3_pmu_module_init(void)
1641 {
1642 int ret;
1643
1644 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
1645 "AP_PERF_ARM_HNS3_PMU_ONLINE",
1646 hns3_pmu_online_cpu,
1647 hns3_pmu_offline_cpu);
1648 if (ret) {
1649 pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret);
1650 return ret;
1651 }
1652
1653 ret = pci_register_driver(&hns3_pmu_driver);
1654 if (ret) {
1655 pr_err("failed to register pci driver, ret = %d.\n", ret);
1656 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
1657 }
1658
1659 return ret;
1660 }
1661 module_init(hns3_pmu_module_init);
1662
hns3_pmu_module_exit(void)1663 static void __exit hns3_pmu_module_exit(void)
1664 {
1665 pci_unregister_driver(&hns3_pmu_driver);
1666 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
1667 }
1668 module_exit(hns3_pmu_module_exit);
1669
1670 MODULE_DESCRIPTION("HNS3 PMU driver");
1671 MODULE_LICENSE("GPL v2");
1672