1 /***********************license start***************
2 * Author: Cavium Networks
3 *
4 * Contact: support@caviumnetworks.com
5 * This file is part of the OCTEON SDK
6 *
7 * Copyright (c) 2003-2017 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more
17 * details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this file; if not, write to the Free Software
21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22 * or visit http://www.gnu.org/licenses/.
23 *
24 * This file may also be available under a different license from Cavium.
25 * Contact Cavium Networks for more information
26 ***********************license end**************************************/
27
28 /*
29 * Implementation of the Level 2 Cache (L2C) control,
30 * measurement, and debugging facilities.
31 */
32
33 #include <linux/compiler.h>
34 #include <linux/irqflags.h>
35 #include <asm/octeon/cvmx.h>
36 #include <asm/octeon/cvmx-l2c.h>
37 #include <asm/octeon/cvmx-spinlock.h>
38
39 /*
40 * This spinlock is used internally to ensure that only one core is
41 * performing certain L2 operations at a time.
42 *
43 * NOTE: This only protects calls from within a single application -
44 * if multiple applications or operating systems are running, then it
45 * is up to the user program to coordinate between them.
46 */
47 static cvmx_spinlock_t cvmx_l2c_spinlock;
48
cvmx_l2c_get_core_way_partition(uint32_t core)49 int cvmx_l2c_get_core_way_partition(uint32_t core)
50 {
51 uint32_t field;
52
53 /* Validate the core number */
54 if (core >= cvmx_octeon_num_cores())
55 return -1;
56
57 if (OCTEON_IS_MODEL(OCTEON_CN63XX))
58 return cvmx_read_csr(CVMX_L2C_WPAR_PPX(core)) & 0xffff;
59
60 /*
61 * Use the lower two bits of the coreNumber to determine the
62 * bit offset of the UMSK[] field in the L2C_SPAR register.
63 */
64 field = (core & 0x3) * 8;
65
66 /*
67 * Return the UMSK[] field from the appropriate L2C_SPAR
68 * register based on the coreNumber.
69 */
70
71 switch (core & 0xC) {
72 case 0x0:
73 return (cvmx_read_csr(CVMX_L2C_SPAR0) & (0xFF << field)) >> field;
74 case 0x4:
75 return (cvmx_read_csr(CVMX_L2C_SPAR1) & (0xFF << field)) >> field;
76 case 0x8:
77 return (cvmx_read_csr(CVMX_L2C_SPAR2) & (0xFF << field)) >> field;
78 case 0xC:
79 return (cvmx_read_csr(CVMX_L2C_SPAR3) & (0xFF << field)) >> field;
80 }
81 return 0;
82 }
83
cvmx_l2c_set_core_way_partition(uint32_t core,uint32_t mask)84 int cvmx_l2c_set_core_way_partition(uint32_t core, uint32_t mask)
85 {
86 uint32_t field;
87 uint32_t valid_mask;
88
89 valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
90
91 mask &= valid_mask;
92
93 /* A UMSK setting which blocks all L2C Ways is an error on some chips */
94 if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
95 return -1;
96
97 /* Validate the core number */
98 if (core >= cvmx_octeon_num_cores())
99 return -1;
100
101 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
102 cvmx_write_csr(CVMX_L2C_WPAR_PPX(core), mask);
103 return 0;
104 }
105
106 /*
107 * Use the lower two bits of core to determine the bit offset of the
108 * UMSK[] field in the L2C_SPAR register.
109 */
110 field = (core & 0x3) * 8;
111
112 /*
113 * Assign the new mask setting to the UMSK[] field in the appropriate
114 * L2C_SPAR register based on the core_num.
115 *
116 */
117 switch (core & 0xC) {
118 case 0x0:
119 cvmx_write_csr(CVMX_L2C_SPAR0,
120 (cvmx_read_csr(CVMX_L2C_SPAR0) & ~(0xFF << field)) |
121 mask << field);
122 break;
123 case 0x4:
124 cvmx_write_csr(CVMX_L2C_SPAR1,
125 (cvmx_read_csr(CVMX_L2C_SPAR1) & ~(0xFF << field)) |
126 mask << field);
127 break;
128 case 0x8:
129 cvmx_write_csr(CVMX_L2C_SPAR2,
130 (cvmx_read_csr(CVMX_L2C_SPAR2) & ~(0xFF << field)) |
131 mask << field);
132 break;
133 case 0xC:
134 cvmx_write_csr(CVMX_L2C_SPAR3,
135 (cvmx_read_csr(CVMX_L2C_SPAR3) & ~(0xFF << field)) |
136 mask << field);
137 break;
138 }
139 return 0;
140 }
141
cvmx_l2c_set_hw_way_partition(uint32_t mask)142 int cvmx_l2c_set_hw_way_partition(uint32_t mask)
143 {
144 uint32_t valid_mask;
145
146 valid_mask = (0x1 << cvmx_l2c_get_num_assoc()) - 1;
147 mask &= valid_mask;
148
149 /* A UMSK setting which blocks all L2C Ways is an error on some chips */
150 if (mask == valid_mask && !OCTEON_IS_MODEL(OCTEON_CN63XX))
151 return -1;
152
153 if (OCTEON_IS_MODEL(OCTEON_CN63XX))
154 cvmx_write_csr(CVMX_L2C_WPAR_IOBX(0), mask);
155 else
156 cvmx_write_csr(CVMX_L2C_SPAR4,
157 (cvmx_read_csr(CVMX_L2C_SPAR4) & ~0xFF) | mask);
158 return 0;
159 }
160
cvmx_l2c_get_hw_way_partition(void)161 int cvmx_l2c_get_hw_way_partition(void)
162 {
163 if (OCTEON_IS_MODEL(OCTEON_CN63XX))
164 return cvmx_read_csr(CVMX_L2C_WPAR_IOBX(0)) & 0xffff;
165 else
166 return cvmx_read_csr(CVMX_L2C_SPAR4) & (0xFF);
167 }
168
cvmx_l2c_config_perf(uint32_t counter,enum cvmx_l2c_event event,uint32_t clear_on_read)169 void cvmx_l2c_config_perf(uint32_t counter, enum cvmx_l2c_event event,
170 uint32_t clear_on_read)
171 {
172 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX)) {
173 union cvmx_l2c_pfctl pfctl;
174
175 pfctl.u64 = cvmx_read_csr(CVMX_L2C_PFCTL);
176
177 switch (counter) {
178 case 0:
179 pfctl.s.cnt0sel = event;
180 pfctl.s.cnt0ena = 1;
181 pfctl.s.cnt0rdclr = clear_on_read;
182 break;
183 case 1:
184 pfctl.s.cnt1sel = event;
185 pfctl.s.cnt1ena = 1;
186 pfctl.s.cnt1rdclr = clear_on_read;
187 break;
188 case 2:
189 pfctl.s.cnt2sel = event;
190 pfctl.s.cnt2ena = 1;
191 pfctl.s.cnt2rdclr = clear_on_read;
192 break;
193 case 3:
194 default:
195 pfctl.s.cnt3sel = event;
196 pfctl.s.cnt3ena = 1;
197 pfctl.s.cnt3rdclr = clear_on_read;
198 break;
199 }
200
201 cvmx_write_csr(CVMX_L2C_PFCTL, pfctl.u64);
202 } else {
203 union cvmx_l2c_tadx_prf l2c_tadx_prf;
204 int tad;
205
206 cvmx_dprintf("L2C performance counter events are different for this chip, mapping 'event' to cvmx_l2c_tad_event_t\n");
207 if (clear_on_read)
208 cvmx_dprintf("L2C counters don't support clear on read for this chip\n");
209
210 l2c_tadx_prf.u64 = cvmx_read_csr(CVMX_L2C_TADX_PRF(0));
211
212 switch (counter) {
213 case 0:
214 l2c_tadx_prf.s.cnt0sel = event;
215 break;
216 case 1:
217 l2c_tadx_prf.s.cnt1sel = event;
218 break;
219 case 2:
220 l2c_tadx_prf.s.cnt2sel = event;
221 break;
222 default:
223 case 3:
224 l2c_tadx_prf.s.cnt3sel = event;
225 break;
226 }
227 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
228 cvmx_write_csr(CVMX_L2C_TADX_PRF(tad),
229 l2c_tadx_prf.u64);
230 }
231 }
232
cvmx_l2c_read_perf(uint32_t counter)233 uint64_t cvmx_l2c_read_perf(uint32_t counter)
234 {
235 switch (counter) {
236 case 0:
237 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
238 return cvmx_read_csr(CVMX_L2C_PFC0);
239 else {
240 uint64_t counter = 0;
241 int tad;
242
243 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
244 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC0(tad));
245 return counter;
246 }
247 case 1:
248 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
249 return cvmx_read_csr(CVMX_L2C_PFC1);
250 else {
251 uint64_t counter = 0;
252 int tad;
253
254 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
255 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC1(tad));
256 return counter;
257 }
258 case 2:
259 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
260 return cvmx_read_csr(CVMX_L2C_PFC2);
261 else {
262 uint64_t counter = 0;
263 int tad;
264
265 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
266 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC2(tad));
267 return counter;
268 }
269 case 3:
270 default:
271 if (OCTEON_IS_MODEL(OCTEON_CN5XXX) || OCTEON_IS_MODEL(OCTEON_CN3XXX))
272 return cvmx_read_csr(CVMX_L2C_PFC3);
273 else {
274 uint64_t counter = 0;
275 int tad;
276
277 for (tad = 0; tad < CVMX_L2C_TADS; tad++)
278 counter += cvmx_read_csr(CVMX_L2C_TADX_PFC3(tad));
279 return counter;
280 }
281 }
282 }
283
284 /*
285 * @INTERNAL
286 * Helper function use to fault in cache lines for L2 cache locking
287 *
288 * @addr: Address of base of memory region to read into L2 cache
289 * @len: Length (in bytes) of region to fault in
290 */
fault_in(uint64_t addr,int len)291 static void fault_in(uint64_t addr, int len)
292 {
293 char *ptr;
294
295 /*
296 * Adjust addr and length so we get all cache lines even for
297 * small ranges spanning two cache lines.
298 */
299 len += addr & CVMX_CACHE_LINE_MASK;
300 addr &= ~CVMX_CACHE_LINE_MASK;
301 ptr = cvmx_phys_to_ptr(addr);
302 /*
303 * Invalidate L1 cache to make sure all loads result in data
304 * being in L2.
305 */
306 CVMX_DCACHE_INVALIDATE;
307 while (len > 0) {
308 READ_ONCE(*ptr);
309 len -= CVMX_CACHE_LINE_SIZE;
310 ptr += CVMX_CACHE_LINE_SIZE;
311 }
312 }
313
cvmx_l2c_lock_line(uint64_t addr)314 int cvmx_l2c_lock_line(uint64_t addr)
315 {
316 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
317 int shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
318 uint64_t assoc = cvmx_l2c_get_num_assoc();
319 uint64_t tag = addr >> shift;
320 uint64_t index = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, cvmx_l2c_address_to_index(addr) << CVMX_L2C_IDX_ADDR_SHIFT);
321 uint64_t way;
322 union cvmx_l2c_tadx_tag l2c_tadx_tag;
323
324 CVMX_CACHE_LCKL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, addr), 0);
325
326 /* Make sure we were able to lock the line */
327 for (way = 0; way < assoc; way++) {
328 CVMX_CACHE_LTGL2I(index | (way << shift), 0);
329 /* make sure CVMX_L2C_TADX_TAG is updated */
330 CVMX_SYNC;
331 l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
332 if (l2c_tadx_tag.s.valid && l2c_tadx_tag.s.tag == tag)
333 break;
334 }
335
336 /* Check if a valid line is found */
337 if (way >= assoc) {
338 /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: line not found for locking at 0x%llx address\n", (unsigned long long)addr); */
339 return -1;
340 }
341
342 /* Check if lock bit is not set */
343 if (!l2c_tadx_tag.s.lock) {
344 /* cvmx_dprintf("ERROR: cvmx_l2c_lock_line: Not able to lock at 0x%llx address\n", (unsigned long long)addr); */
345 return -1;
346 }
347 return way;
348 } else {
349 int retval = 0;
350 union cvmx_l2c_dbg l2cdbg;
351 union cvmx_l2c_lckbase lckbase;
352 union cvmx_l2c_lckoff lckoff;
353 union cvmx_l2t_err l2t_err;
354
355 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
356
357 l2cdbg.u64 = 0;
358 lckbase.u64 = 0;
359 lckoff.u64 = 0;
360
361 /* Clear l2t error bits if set */
362 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
363 l2t_err.s.lckerr = 1;
364 l2t_err.s.lckerr2 = 1;
365 cvmx_write_csr(CVMX_L2T_ERR, l2t_err.u64);
366
367 addr &= ~CVMX_CACHE_LINE_MASK;
368
369 /* Set this core as debug core */
370 l2cdbg.s.ppnum = cvmx_get_core_num();
371 CVMX_SYNC;
372 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
373 cvmx_read_csr(CVMX_L2C_DBG);
374
375 lckoff.s.lck_offset = 0; /* Only lock 1 line at a time */
376 cvmx_write_csr(CVMX_L2C_LCKOFF, lckoff.u64);
377 cvmx_read_csr(CVMX_L2C_LCKOFF);
378
379 if (((union cvmx_l2c_cfg)(cvmx_read_csr(CVMX_L2C_CFG))).s.idxalias) {
380 int alias_shift = CVMX_L2C_IDX_ADDR_SHIFT + 2 * CVMX_L2_SET_BITS - 1;
381 uint64_t addr_tmp = addr ^ (addr & ((1 << alias_shift) - 1)) >> CVMX_L2_SET_BITS;
382
383 lckbase.s.lck_base = addr_tmp >> 7;
384
385 } else {
386 lckbase.s.lck_base = addr >> 7;
387 }
388
389 lckbase.s.lck_ena = 1;
390 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
391 /* Make sure it gets there */
392 cvmx_read_csr(CVMX_L2C_LCKBASE);
393
394 fault_in(addr, CVMX_CACHE_LINE_SIZE);
395
396 lckbase.s.lck_ena = 0;
397 cvmx_write_csr(CVMX_L2C_LCKBASE, lckbase.u64);
398 /* Make sure it gets there */
399 cvmx_read_csr(CVMX_L2C_LCKBASE);
400
401 /* Stop being debug core */
402 cvmx_write_csr(CVMX_L2C_DBG, 0);
403 cvmx_read_csr(CVMX_L2C_DBG);
404
405 l2t_err.u64 = cvmx_read_csr(CVMX_L2T_ERR);
406 if (l2t_err.s.lckerr || l2t_err.s.lckerr2)
407 retval = 1; /* We were unable to lock the line */
408
409 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
410 return retval;
411 }
412 }
413
cvmx_l2c_lock_mem_region(uint64_t start,uint64_t len)414 int cvmx_l2c_lock_mem_region(uint64_t start, uint64_t len)
415 {
416 int retval = 0;
417
418 /* Round start/end to cache line boundaries */
419 len += start & CVMX_CACHE_LINE_MASK;
420 start &= ~CVMX_CACHE_LINE_MASK;
421 len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
422
423 while (len) {
424 retval += cvmx_l2c_lock_line(start);
425 start += CVMX_CACHE_LINE_SIZE;
426 len -= CVMX_CACHE_LINE_SIZE;
427 }
428 return retval;
429 }
430
cvmx_l2c_flush(void)431 void cvmx_l2c_flush(void)
432 {
433 uint64_t assoc, set;
434 uint64_t n_assoc, n_set;
435
436 n_set = cvmx_l2c_get_num_sets();
437 n_assoc = cvmx_l2c_get_num_assoc();
438
439 if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
440 uint64_t address;
441 /* These may look like constants, but they aren't... */
442 int assoc_shift = CVMX_L2C_TAG_ADDR_ALIAS_SHIFT;
443 int set_shift = CVMX_L2C_IDX_ADDR_SHIFT;
444
445 for (set = 0; set < n_set; set++) {
446 for (assoc = 0; assoc < n_assoc; assoc++) {
447 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
448 (assoc << assoc_shift) | (set << set_shift));
449 CVMX_CACHE_WBIL2I(address, 0);
450 }
451 }
452 } else {
453 for (set = 0; set < n_set; set++)
454 for (assoc = 0; assoc < n_assoc; assoc++)
455 cvmx_l2c_flush_line(assoc, set);
456 }
457 }
458
459
cvmx_l2c_unlock_line(uint64_t address)460 int cvmx_l2c_unlock_line(uint64_t address)
461 {
462
463 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
464 int assoc;
465 union cvmx_l2c_tag tag;
466 uint32_t tag_addr;
467 uint32_t index = cvmx_l2c_address_to_index(address);
468
469 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
470
471 /*
472 * For 63XX, we can flush a line by using the physical
473 * address directly, so finding the cache line used by
474 * the address is only required to provide the proper
475 * return value for the function.
476 */
477 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
478 tag = cvmx_l2c_get_tag(assoc, index);
479
480 if (tag.s.V && (tag.s.addr == tag_addr)) {
481 CVMX_CACHE_WBIL2(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, address), 0);
482 return tag.s.L;
483 }
484 }
485 } else {
486 int assoc;
487 union cvmx_l2c_tag tag;
488 uint32_t tag_addr;
489
490 uint32_t index = cvmx_l2c_address_to_index(address);
491
492 /* Compute portion of address that is stored in tag */
493 tag_addr = ((address >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) & ((1 << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) - 1));
494 for (assoc = 0; assoc < CVMX_L2_ASSOC; assoc++) {
495 tag = cvmx_l2c_get_tag(assoc, index);
496
497 if (tag.s.V && (tag.s.addr == tag_addr)) {
498 cvmx_l2c_flush_line(assoc, index);
499 return tag.s.L;
500 }
501 }
502 }
503 return 0;
504 }
505
cvmx_l2c_unlock_mem_region(uint64_t start,uint64_t len)506 int cvmx_l2c_unlock_mem_region(uint64_t start, uint64_t len)
507 {
508 int num_unlocked = 0;
509 /* Round start/end to cache line boundaries */
510 len += start & CVMX_CACHE_LINE_MASK;
511 start &= ~CVMX_CACHE_LINE_MASK;
512 len = (len + CVMX_CACHE_LINE_MASK) & ~CVMX_CACHE_LINE_MASK;
513 while (len > 0) {
514 num_unlocked += cvmx_l2c_unlock_line(start);
515 start += CVMX_CACHE_LINE_SIZE;
516 len -= CVMX_CACHE_LINE_SIZE;
517 }
518
519 return num_unlocked;
520 }
521
522 /*
523 * Internal l2c tag types. These are converted to a generic structure
524 * that can be used on all chips.
525 */
526 union __cvmx_l2c_tag {
527 uint64_t u64;
528 struct cvmx_l2c_tag_cn50xx {
529 __BITFIELD_FIELD(uint64_t reserved:40,
530 __BITFIELD_FIELD(uint64_t V:1, /* Line valid */
531 __BITFIELD_FIELD(uint64_t D:1, /* Line dirty */
532 __BITFIELD_FIELD(uint64_t L:1, /* Line locked */
533 __BITFIELD_FIELD(uint64_t U:1, /* Use, LRU eviction */
534 __BITFIELD_FIELD(uint64_t addr:20, /* Phys addr (33..14) */
535 ;))))))
536 } cn50xx;
537 struct cvmx_l2c_tag_cn30xx {
538 __BITFIELD_FIELD(uint64_t reserved:41,
539 __BITFIELD_FIELD(uint64_t V:1, /* Line valid */
540 __BITFIELD_FIELD(uint64_t D:1, /* Line dirty */
541 __BITFIELD_FIELD(uint64_t L:1, /* Line locked */
542 __BITFIELD_FIELD(uint64_t U:1, /* Use, LRU eviction */
543 __BITFIELD_FIELD(uint64_t addr:19, /* Phys addr (33..15) */
544 ;))))))
545 } cn30xx;
546 struct cvmx_l2c_tag_cn31xx {
547 __BITFIELD_FIELD(uint64_t reserved:42,
548 __BITFIELD_FIELD(uint64_t V:1, /* Line valid */
549 __BITFIELD_FIELD(uint64_t D:1, /* Line dirty */
550 __BITFIELD_FIELD(uint64_t L:1, /* Line locked */
551 __BITFIELD_FIELD(uint64_t U:1, /* Use, LRU eviction */
552 __BITFIELD_FIELD(uint64_t addr:18, /* Phys addr (33..16) */
553 ;))))))
554 } cn31xx;
555 struct cvmx_l2c_tag_cn38xx {
556 __BITFIELD_FIELD(uint64_t reserved:43,
557 __BITFIELD_FIELD(uint64_t V:1, /* Line valid */
558 __BITFIELD_FIELD(uint64_t D:1, /* Line dirty */
559 __BITFIELD_FIELD(uint64_t L:1, /* Line locked */
560 __BITFIELD_FIELD(uint64_t U:1, /* Use, LRU eviction */
561 __BITFIELD_FIELD(uint64_t addr:17, /* Phys addr (33..17) */
562 ;))))))
563 } cn38xx;
564 struct cvmx_l2c_tag_cn58xx {
565 __BITFIELD_FIELD(uint64_t reserved:44,
566 __BITFIELD_FIELD(uint64_t V:1, /* Line valid */
567 __BITFIELD_FIELD(uint64_t D:1, /* Line dirty */
568 __BITFIELD_FIELD(uint64_t L:1, /* Line locked */
569 __BITFIELD_FIELD(uint64_t U:1, /* Use, LRU eviction */
570 __BITFIELD_FIELD(uint64_t addr:16, /* Phys addr (33..18) */
571 ;))))))
572 } cn58xx;
573 struct cvmx_l2c_tag_cn58xx cn56xx; /* 2048 sets */
574 struct cvmx_l2c_tag_cn31xx cn52xx; /* 512 sets */
575 };
576
577
578 /*
579 * @INTERNAL
580 * Function to read a L2C tag. This code make the current core
581 * the 'debug core' for the L2. This code must only be executed by
582 * 1 core at a time.
583 *
584 * @assoc: Association (way) of the tag to dump
585 * @index: Index of the cacheline
586 *
587 * Returns The Octeon model specific tag structure. This is
588 * translated by a wrapper function to a generic form that is
589 * easier for applications to use.
590 */
__read_l2_tag(uint64_t assoc,uint64_t index)591 static union __cvmx_l2c_tag __read_l2_tag(uint64_t assoc, uint64_t index)
592 {
593
594 uint64_t debug_tag_addr = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS, (index << 7) + 96);
595 uint64_t core = cvmx_get_core_num();
596 union __cvmx_l2c_tag tag_val;
597 uint64_t dbg_addr = CVMX_L2C_DBG;
598 unsigned long flags;
599 union cvmx_l2c_dbg debug_val;
600
601 debug_val.u64 = 0;
602 /*
603 * For low core count parts, the core number is always small
604 * enough to stay in the correct field and not set any
605 * reserved bits.
606 */
607 debug_val.s.ppnum = core;
608 debug_val.s.l2t = 1;
609 debug_val.s.set = assoc;
610
611 local_irq_save(flags);
612 /*
613 * Make sure core is quiet (no prefetches, etc.) before
614 * entering debug mode.
615 */
616 CVMX_SYNC;
617 /* Flush L1 to make sure debug load misses L1 */
618 CVMX_DCACHE_INVALIDATE;
619
620 /*
621 * The following must be done in assembly as when in debug
622 * mode all data loads from L2 return special debug data, not
623 * normal memory contents. Also, interrupts must be disabled,
624 * since if an interrupt occurs while in debug mode the ISR
625 * will get debug data from all its memory * reads instead of
626 * the contents of memory.
627 */
628
629 asm volatile (
630 ".set push\n\t"
631 ".set mips64\n\t"
632 ".set noreorder\n\t"
633 "sd %[dbg_val], 0(%[dbg_addr])\n\t" /* Enter debug mode, wait for store */
634 "ld $0, 0(%[dbg_addr])\n\t"
635 "ld %[tag_val], 0(%[tag_addr])\n\t" /* Read L2C tag data */
636 "sd $0, 0(%[dbg_addr])\n\t" /* Exit debug mode, wait for store */
637 "ld $0, 0(%[dbg_addr])\n\t"
638 "cache 9, 0($0)\n\t" /* Invalidate dcache to discard debug data */
639 ".set pop"
640 : [tag_val] "=r" (tag_val)
641 : [dbg_addr] "r" (dbg_addr), [dbg_val] "r" (debug_val), [tag_addr] "r" (debug_tag_addr)
642 : "memory");
643
644 local_irq_restore(flags);
645
646 return tag_val;
647 }
648
649
cvmx_l2c_get_tag(uint32_t association,uint32_t index)650 union cvmx_l2c_tag cvmx_l2c_get_tag(uint32_t association, uint32_t index)
651 {
652 union cvmx_l2c_tag tag;
653
654 tag.u64 = 0;
655 if ((int)association >= cvmx_l2c_get_num_assoc()) {
656 cvmx_dprintf("ERROR: cvmx_l2c_get_tag association out of range\n");
657 return tag;
658 }
659 if ((int)index >= cvmx_l2c_get_num_sets()) {
660 cvmx_dprintf("ERROR: cvmx_l2c_get_tag index out of range (arg: %d, max: %d)\n",
661 (int)index, cvmx_l2c_get_num_sets());
662 return tag;
663 }
664 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
665 union cvmx_l2c_tadx_tag l2c_tadx_tag;
666 uint64_t address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
667 (association << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
668 (index << CVMX_L2C_IDX_ADDR_SHIFT));
669 /*
670 * Use L2 cache Index load tag cache instruction, as
671 * hardware loads the virtual tag for the L2 cache
672 * block with the contents of L2C_TAD0_TAG
673 * register.
674 */
675 CVMX_CACHE_LTGL2I(address, 0);
676 CVMX_SYNC; /* make sure CVMX_L2C_TADX_TAG is updated */
677 l2c_tadx_tag.u64 = cvmx_read_csr(CVMX_L2C_TADX_TAG(0));
678
679 tag.s.V = l2c_tadx_tag.s.valid;
680 tag.s.D = l2c_tadx_tag.s.dirty;
681 tag.s.L = l2c_tadx_tag.s.lock;
682 tag.s.U = l2c_tadx_tag.s.use;
683 tag.s.addr = l2c_tadx_tag.s.tag;
684 } else {
685 union __cvmx_l2c_tag tmp_tag;
686 /* __read_l2_tag is intended for internal use only */
687 tmp_tag = __read_l2_tag(association, index);
688
689 /*
690 * Convert all tag structure types to generic version,
691 * as it can represent all models.
692 */
693 if (OCTEON_IS_MODEL(OCTEON_CN58XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
694 tag.s.V = tmp_tag.cn58xx.V;
695 tag.s.D = tmp_tag.cn58xx.D;
696 tag.s.L = tmp_tag.cn58xx.L;
697 tag.s.U = tmp_tag.cn58xx.U;
698 tag.s.addr = tmp_tag.cn58xx.addr;
699 } else if (OCTEON_IS_MODEL(OCTEON_CN38XX)) {
700 tag.s.V = tmp_tag.cn38xx.V;
701 tag.s.D = tmp_tag.cn38xx.D;
702 tag.s.L = tmp_tag.cn38xx.L;
703 tag.s.U = tmp_tag.cn38xx.U;
704 tag.s.addr = tmp_tag.cn38xx.addr;
705 } else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX)) {
706 tag.s.V = tmp_tag.cn31xx.V;
707 tag.s.D = tmp_tag.cn31xx.D;
708 tag.s.L = tmp_tag.cn31xx.L;
709 tag.s.U = tmp_tag.cn31xx.U;
710 tag.s.addr = tmp_tag.cn31xx.addr;
711 } else if (OCTEON_IS_MODEL(OCTEON_CN30XX)) {
712 tag.s.V = tmp_tag.cn30xx.V;
713 tag.s.D = tmp_tag.cn30xx.D;
714 tag.s.L = tmp_tag.cn30xx.L;
715 tag.s.U = tmp_tag.cn30xx.U;
716 tag.s.addr = tmp_tag.cn30xx.addr;
717 } else if (OCTEON_IS_MODEL(OCTEON_CN50XX)) {
718 tag.s.V = tmp_tag.cn50xx.V;
719 tag.s.D = tmp_tag.cn50xx.D;
720 tag.s.L = tmp_tag.cn50xx.L;
721 tag.s.U = tmp_tag.cn50xx.U;
722 tag.s.addr = tmp_tag.cn50xx.addr;
723 } else {
724 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
725 }
726 }
727 return tag;
728 }
729
cvmx_l2c_address_to_index(uint64_t addr)730 uint32_t cvmx_l2c_address_to_index(uint64_t addr)
731 {
732 uint64_t idx = addr >> CVMX_L2C_IDX_ADDR_SHIFT;
733 int indxalias = 0;
734
735 if (OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
736 union cvmx_l2c_ctl l2c_ctl;
737
738 l2c_ctl.u64 = cvmx_read_csr(CVMX_L2C_CTL);
739 indxalias = !l2c_ctl.s.disidxalias;
740 } else {
741 union cvmx_l2c_cfg l2c_cfg;
742
743 l2c_cfg.u64 = cvmx_read_csr(CVMX_L2C_CFG);
744 indxalias = l2c_cfg.s.idxalias;
745 }
746
747 if (indxalias) {
748 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
749 uint32_t a_14_12 = (idx / (CVMX_L2C_MEMBANK_SELECT_SIZE/(1<<CVMX_L2C_IDX_ADDR_SHIFT))) & 0x7;
750
751 idx ^= idx / cvmx_l2c_get_num_sets();
752 idx ^= a_14_12;
753 } else {
754 idx ^= ((addr & CVMX_L2C_ALIAS_MASK) >> CVMX_L2C_TAG_ADDR_ALIAS_SHIFT);
755 }
756 }
757 idx &= CVMX_L2C_IDX_MASK;
758 return idx;
759 }
760
cvmx_l2c_get_cache_size_bytes(void)761 int cvmx_l2c_get_cache_size_bytes(void)
762 {
763 return cvmx_l2c_get_num_sets() * cvmx_l2c_get_num_assoc() *
764 CVMX_CACHE_LINE_SIZE;
765 }
766
767 /*
768 * Return log base 2 of the number of sets in the L2 cache
769 */
cvmx_l2c_get_set_bits(void)770 int cvmx_l2c_get_set_bits(void)
771 {
772 int l2_set_bits;
773
774 if (OCTEON_IS_MODEL(OCTEON_CN56XX) || OCTEON_IS_MODEL(OCTEON_CN58XX))
775 l2_set_bits = 11; /* 2048 sets */
776 else if (OCTEON_IS_MODEL(OCTEON_CN38XX) || OCTEON_IS_MODEL(OCTEON_CN63XX))
777 l2_set_bits = 10; /* 1024 sets */
778 else if (OCTEON_IS_MODEL(OCTEON_CN31XX) || OCTEON_IS_MODEL(OCTEON_CN52XX))
779 l2_set_bits = 9; /* 512 sets */
780 else if (OCTEON_IS_MODEL(OCTEON_CN30XX))
781 l2_set_bits = 8; /* 256 sets */
782 else if (OCTEON_IS_MODEL(OCTEON_CN50XX))
783 l2_set_bits = 7; /* 128 sets */
784 else {
785 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
786 l2_set_bits = 11; /* 2048 sets */
787 }
788 return l2_set_bits;
789 }
790
791 /* Return the number of sets in the L2 Cache */
cvmx_l2c_get_num_sets(void)792 int cvmx_l2c_get_num_sets(void)
793 {
794 return 1 << cvmx_l2c_get_set_bits();
795 }
796
797 /* Return the number of associations in the L2 Cache */
cvmx_l2c_get_num_assoc(void)798 int cvmx_l2c_get_num_assoc(void)
799 {
800 int l2_assoc;
801
802 if (OCTEON_IS_MODEL(OCTEON_CN56XX) ||
803 OCTEON_IS_MODEL(OCTEON_CN52XX) ||
804 OCTEON_IS_MODEL(OCTEON_CN58XX) ||
805 OCTEON_IS_MODEL(OCTEON_CN50XX) ||
806 OCTEON_IS_MODEL(OCTEON_CN38XX))
807 l2_assoc = 8;
808 else if (OCTEON_IS_MODEL(OCTEON_CN63XX))
809 l2_assoc = 16;
810 else if (OCTEON_IS_MODEL(OCTEON_CN31XX) ||
811 OCTEON_IS_MODEL(OCTEON_CN30XX))
812 l2_assoc = 4;
813 else {
814 cvmx_dprintf("Unsupported OCTEON Model in %s\n", __func__);
815 l2_assoc = 8;
816 }
817
818 /* Check to see if part of the cache is disabled */
819 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
820 union cvmx_mio_fus_dat3 mio_fus_dat3;
821
822 mio_fus_dat3.u64 = cvmx_read_csr(CVMX_MIO_FUS_DAT3);
823 /*
824 * cvmx_mio_fus_dat3.s.l2c_crip fuses map as follows
825 * <2> will be not used for 63xx
826 * <1> disables 1/2 ways
827 * <0> disables 1/4 ways
828 * They are cumulative, so for 63xx:
829 * <1> <0>
830 * 0 0 16-way 2MB cache
831 * 0 1 12-way 1.5MB cache
832 * 1 0 8-way 1MB cache
833 * 1 1 4-way 512KB cache
834 */
835
836 if (mio_fus_dat3.s.l2c_crip == 3)
837 l2_assoc = 4;
838 else if (mio_fus_dat3.s.l2c_crip == 2)
839 l2_assoc = 8;
840 else if (mio_fus_dat3.s.l2c_crip == 1)
841 l2_assoc = 12;
842 } else {
843 uint64_t l2d_fus3;
844
845 l2d_fus3 = cvmx_read_csr(CVMX_L2D_FUS3);
846 /*
847 * Using shifts here, as bit position names are
848 * different for each model but they all mean the
849 * same.
850 */
851 if ((l2d_fus3 >> 35) & 0x1)
852 l2_assoc = l2_assoc >> 2;
853 else if ((l2d_fus3 >> 34) & 0x1)
854 l2_assoc = l2_assoc >> 1;
855 }
856 return l2_assoc;
857 }
858
859 /*
860 * Flush a line from the L2 cache
861 * This should only be called from one core at a time, as this routine
862 * sets the core to the 'debug' core in order to flush the line.
863 *
864 * @assoc: Association (or way) to flush
865 * @index: Index to flush
866 */
cvmx_l2c_flush_line(uint32_t assoc,uint32_t index)867 void cvmx_l2c_flush_line(uint32_t assoc, uint32_t index)
868 {
869 /* Check the range of the index. */
870 if (index > (uint32_t)cvmx_l2c_get_num_sets()) {
871 cvmx_dprintf("ERROR: cvmx_l2c_flush_line index out of range.\n");
872 return;
873 }
874
875 /* Check the range of association. */
876 if (assoc > (uint32_t)cvmx_l2c_get_num_assoc()) {
877 cvmx_dprintf("ERROR: cvmx_l2c_flush_line association out of range.\n");
878 return;
879 }
880
881 if (OCTEON_IS_MODEL(OCTEON_CN63XX)) {
882 uint64_t address;
883 /* Create the address based on index and association.
884 * Bits<20:17> select the way of the cache block involved in
885 * the operation
886 * Bits<16:7> of the effect address select the index
887 */
888 address = CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
889 (assoc << CVMX_L2C_TAG_ADDR_ALIAS_SHIFT) |
890 (index << CVMX_L2C_IDX_ADDR_SHIFT));
891 CVMX_CACHE_WBIL2I(address, 0);
892 } else {
893 union cvmx_l2c_dbg l2cdbg;
894
895 l2cdbg.u64 = 0;
896 if (!OCTEON_IS_MODEL(OCTEON_CN30XX))
897 l2cdbg.s.ppnum = cvmx_get_core_num();
898 l2cdbg.s.finv = 1;
899
900 l2cdbg.s.set = assoc;
901 cvmx_spinlock_lock(&cvmx_l2c_spinlock);
902 /*
903 * Enter debug mode, and make sure all other writes
904 * complete before we enter debug mode
905 */
906 CVMX_SYNC;
907 cvmx_write_csr(CVMX_L2C_DBG, l2cdbg.u64);
908 cvmx_read_csr(CVMX_L2C_DBG);
909
910 CVMX_PREPARE_FOR_STORE(CVMX_ADD_SEG(CVMX_MIPS_SPACE_XKPHYS,
911 index * CVMX_CACHE_LINE_SIZE),
912 0);
913 /* Exit debug mode */
914 CVMX_SYNC;
915 cvmx_write_csr(CVMX_L2C_DBG, 0);
916 cvmx_read_csr(CVMX_L2C_DBG);
917 cvmx_spinlock_unlock(&cvmx_l2c_spinlock);
918 }
919 }
920