1 /******************************************************************************
2 * arch/ia64/kernel/paravirt.c
3 *
4 * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
5 * VA Linux Systems Japan K.K.
6 * Yaozu (Eddie) Dong <eddie.dong@intel.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 *
22 */
23
24 #include <linux/init.h>
25
26 #include <linux/compiler.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/module.h>
30 #include <linux/types.h>
31
32 #include <asm/iosapic.h>
33 #include <asm/paravirt.h>
34
35 /***************************************************************************
36 * general info
37 */
38 struct pv_info pv_info = {
39 .kernel_rpl = 0,
40 .paravirt_enabled = 0,
41 .name = "bare hardware"
42 };
43
44 /***************************************************************************
45 * pv_init_ops
46 * initialization hooks.
47 */
48
49 static void __init
50 ia64_native_patch_branch(unsigned long tag, unsigned long type);
51
52 struct pv_init_ops pv_init_ops =
53 {
54 #ifdef ASM_SUPPORTED
55 .patch_bundle = ia64_native_patch_bundle,
56 #endif
57 .patch_branch = ia64_native_patch_branch,
58 };
59
60 /***************************************************************************
61 * pv_cpu_ops
62 * intrinsics hooks.
63 */
64
65 #ifndef ASM_SUPPORTED
66 /* ia64_native_xxx are macros so that we have to make them real functions */
67
68 #define DEFINE_VOID_FUNC1(name) \
69 static void \
70 ia64_native_ ## name ## _func(unsigned long arg) \
71 { \
72 ia64_native_ ## name(arg); \
73 }
74
75 #define DEFINE_VOID_FUNC1_VOID(name) \
76 static void \
77 ia64_native_ ## name ## _func(void *arg) \
78 { \
79 ia64_native_ ## name(arg); \
80 }
81
82 #define DEFINE_VOID_FUNC2(name) \
83 static void \
84 ia64_native_ ## name ## _func(unsigned long arg0, \
85 unsigned long arg1) \
86 { \
87 ia64_native_ ## name(arg0, arg1); \
88 }
89
90 #define DEFINE_FUNC0(name) \
91 static unsigned long \
92 ia64_native_ ## name ## _func(void) \
93 { \
94 return ia64_native_ ## name(); \
95 }
96
97 #define DEFINE_FUNC1(name, type) \
98 static unsigned long \
99 ia64_native_ ## name ## _func(type arg) \
100 { \
101 return ia64_native_ ## name(arg); \
102 } \
103
104 DEFINE_VOID_FUNC1_VOID(fc);
105 DEFINE_VOID_FUNC1(intrin_local_irq_restore);
106
107 DEFINE_VOID_FUNC2(ptcga);
108 DEFINE_VOID_FUNC2(set_rr);
109
110 DEFINE_FUNC0(get_psr_i);
111
112 DEFINE_FUNC1(thash, unsigned long);
113 DEFINE_FUNC1(get_cpuid, int);
114 DEFINE_FUNC1(get_pmd, int);
115 DEFINE_FUNC1(get_rr, unsigned long);
116
117 static void
ia64_native_ssm_i_func(void)118 ia64_native_ssm_i_func(void)
119 {
120 ia64_native_ssm(IA64_PSR_I);
121 }
122
123 static void
ia64_native_rsm_i_func(void)124 ia64_native_rsm_i_func(void)
125 {
126 ia64_native_rsm(IA64_PSR_I);
127 }
128
129 static void
ia64_native_set_rr0_to_rr4_func(unsigned long val0,unsigned long val1,unsigned long val2,unsigned long val3,unsigned long val4)130 ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
131 unsigned long val2, unsigned long val3,
132 unsigned long val4)
133 {
134 ia64_native_set_rr0_to_rr4(val0, val1, val2, val3, val4);
135 }
136
137 #define CASE_GET_REG(id) \
138 case _IA64_REG_ ## id: \
139 res = ia64_native_getreg(_IA64_REG_ ## id); \
140 break;
141 #define CASE_GET_AR(id) CASE_GET_REG(AR_ ## id)
142 #define CASE_GET_CR(id) CASE_GET_REG(CR_ ## id)
143
144 unsigned long
ia64_native_getreg_func(int regnum)145 ia64_native_getreg_func(int regnum)
146 {
147 unsigned long res = -1;
148 switch (regnum) {
149 CASE_GET_REG(GP);
150 /*CASE_GET_REG(IP);*/ /* returned ip value shouldn't be constant */
151 CASE_GET_REG(PSR);
152 CASE_GET_REG(TP);
153 CASE_GET_REG(SP);
154
155 CASE_GET_AR(KR0);
156 CASE_GET_AR(KR1);
157 CASE_GET_AR(KR2);
158 CASE_GET_AR(KR3);
159 CASE_GET_AR(KR4);
160 CASE_GET_AR(KR5);
161 CASE_GET_AR(KR6);
162 CASE_GET_AR(KR7);
163 CASE_GET_AR(RSC);
164 CASE_GET_AR(BSP);
165 CASE_GET_AR(BSPSTORE);
166 CASE_GET_AR(RNAT);
167 CASE_GET_AR(FCR);
168 CASE_GET_AR(EFLAG);
169 CASE_GET_AR(CSD);
170 CASE_GET_AR(SSD);
171 CASE_GET_AR(CFLAG);
172 CASE_GET_AR(FSR);
173 CASE_GET_AR(FIR);
174 CASE_GET_AR(FDR);
175 CASE_GET_AR(CCV);
176 CASE_GET_AR(UNAT);
177 CASE_GET_AR(FPSR);
178 CASE_GET_AR(ITC);
179 CASE_GET_AR(PFS);
180 CASE_GET_AR(LC);
181 CASE_GET_AR(EC);
182
183 CASE_GET_CR(DCR);
184 CASE_GET_CR(ITM);
185 CASE_GET_CR(IVA);
186 CASE_GET_CR(PTA);
187 CASE_GET_CR(IPSR);
188 CASE_GET_CR(ISR);
189 CASE_GET_CR(IIP);
190 CASE_GET_CR(IFA);
191 CASE_GET_CR(ITIR);
192 CASE_GET_CR(IIPA);
193 CASE_GET_CR(IFS);
194 CASE_GET_CR(IIM);
195 CASE_GET_CR(IHA);
196 CASE_GET_CR(LID);
197 CASE_GET_CR(IVR);
198 CASE_GET_CR(TPR);
199 CASE_GET_CR(EOI);
200 CASE_GET_CR(IRR0);
201 CASE_GET_CR(IRR1);
202 CASE_GET_CR(IRR2);
203 CASE_GET_CR(IRR3);
204 CASE_GET_CR(ITV);
205 CASE_GET_CR(PMV);
206 CASE_GET_CR(CMCV);
207 CASE_GET_CR(LRR0);
208 CASE_GET_CR(LRR1);
209
210 default:
211 printk(KERN_CRIT "wrong_getreg %d\n", regnum);
212 break;
213 }
214 return res;
215 }
216
217 #define CASE_SET_REG(id) \
218 case _IA64_REG_ ## id: \
219 ia64_native_setreg(_IA64_REG_ ## id, val); \
220 break;
221 #define CASE_SET_AR(id) CASE_SET_REG(AR_ ## id)
222 #define CASE_SET_CR(id) CASE_SET_REG(CR_ ## id)
223
224 void
ia64_native_setreg_func(int regnum,unsigned long val)225 ia64_native_setreg_func(int regnum, unsigned long val)
226 {
227 switch (regnum) {
228 case _IA64_REG_PSR_L:
229 ia64_native_setreg(_IA64_REG_PSR_L, val);
230 ia64_dv_serialize_data();
231 break;
232 CASE_SET_REG(SP);
233 CASE_SET_REG(GP);
234
235 CASE_SET_AR(KR0);
236 CASE_SET_AR(KR1);
237 CASE_SET_AR(KR2);
238 CASE_SET_AR(KR3);
239 CASE_SET_AR(KR4);
240 CASE_SET_AR(KR5);
241 CASE_SET_AR(KR6);
242 CASE_SET_AR(KR7);
243 CASE_SET_AR(RSC);
244 CASE_SET_AR(BSP);
245 CASE_SET_AR(BSPSTORE);
246 CASE_SET_AR(RNAT);
247 CASE_SET_AR(FCR);
248 CASE_SET_AR(EFLAG);
249 CASE_SET_AR(CSD);
250 CASE_SET_AR(SSD);
251 CASE_SET_AR(CFLAG);
252 CASE_SET_AR(FSR);
253 CASE_SET_AR(FIR);
254 CASE_SET_AR(FDR);
255 CASE_SET_AR(CCV);
256 CASE_SET_AR(UNAT);
257 CASE_SET_AR(FPSR);
258 CASE_SET_AR(ITC);
259 CASE_SET_AR(PFS);
260 CASE_SET_AR(LC);
261 CASE_SET_AR(EC);
262
263 CASE_SET_CR(DCR);
264 CASE_SET_CR(ITM);
265 CASE_SET_CR(IVA);
266 CASE_SET_CR(PTA);
267 CASE_SET_CR(IPSR);
268 CASE_SET_CR(ISR);
269 CASE_SET_CR(IIP);
270 CASE_SET_CR(IFA);
271 CASE_SET_CR(ITIR);
272 CASE_SET_CR(IIPA);
273 CASE_SET_CR(IFS);
274 CASE_SET_CR(IIM);
275 CASE_SET_CR(IHA);
276 CASE_SET_CR(LID);
277 CASE_SET_CR(IVR);
278 CASE_SET_CR(TPR);
279 CASE_SET_CR(EOI);
280 CASE_SET_CR(IRR0);
281 CASE_SET_CR(IRR1);
282 CASE_SET_CR(IRR2);
283 CASE_SET_CR(IRR3);
284 CASE_SET_CR(ITV);
285 CASE_SET_CR(PMV);
286 CASE_SET_CR(CMCV);
287 CASE_SET_CR(LRR0);
288 CASE_SET_CR(LRR1);
289 default:
290 printk(KERN_CRIT "wrong setreg %d\n", regnum);
291 break;
292 }
293 }
294 #else
295
296 #define __DEFINE_FUNC(name, code) \
297 extern const char ia64_native_ ## name ## _direct_start[]; \
298 extern const char ia64_native_ ## name ## _direct_end[]; \
299 asm (".align 32\n" \
300 ".proc ia64_native_" #name "_func\n" \
301 "ia64_native_" #name "_func:\n" \
302 "ia64_native_" #name "_direct_start:\n" \
303 code \
304 "ia64_native_" #name "_direct_end:\n" \
305 "br.cond.sptk.many b6\n" \
306 ".endp ia64_native_" #name "_func\n")
307
308 #define DEFINE_VOID_FUNC0(name, code) \
309 extern void \
310 ia64_native_ ## name ## _func(void); \
311 __DEFINE_FUNC(name, code)
312
313 #define DEFINE_VOID_FUNC1(name, code) \
314 extern void \
315 ia64_native_ ## name ## _func(unsigned long arg); \
316 __DEFINE_FUNC(name, code)
317
318 #define DEFINE_VOID_FUNC1_VOID(name, code) \
319 extern void \
320 ia64_native_ ## name ## _func(void *arg); \
321 __DEFINE_FUNC(name, code)
322
323 #define DEFINE_VOID_FUNC2(name, code) \
324 extern void \
325 ia64_native_ ## name ## _func(unsigned long arg0, \
326 unsigned long arg1); \
327 __DEFINE_FUNC(name, code)
328
329 #define DEFINE_FUNC0(name, code) \
330 extern unsigned long \
331 ia64_native_ ## name ## _func(void); \
332 __DEFINE_FUNC(name, code)
333
334 #define DEFINE_FUNC1(name, type, code) \
335 extern unsigned long \
336 ia64_native_ ## name ## _func(type arg); \
337 __DEFINE_FUNC(name, code)
338
339 DEFINE_VOID_FUNC1_VOID(fc,
340 "fc r8\n");
341 DEFINE_VOID_FUNC1(intrin_local_irq_restore,
342 ";;\n"
343 " cmp.ne p6, p7 = r8, r0\n"
344 ";;\n"
345 "(p6) ssm psr.i\n"
346 "(p7) rsm psr.i\n"
347 ";;\n"
348 "(p6) srlz.d\n");
349
350 DEFINE_VOID_FUNC2(ptcga,
351 "ptc.ga r8, r9\n");
352 DEFINE_VOID_FUNC2(set_rr,
353 "mov rr[r8] = r9\n");
354
355 /* ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I */
356 DEFINE_FUNC0(get_psr_i,
357 "mov r2 = " __stringify(1 << IA64_PSR_I_BIT) "\n"
358 "mov r8 = psr\n"
359 ";;\n"
360 "and r8 = r2, r8\n");
361
362 DEFINE_FUNC1(thash, unsigned long,
363 "thash r8 = r8\n");
364 DEFINE_FUNC1(get_cpuid, int,
365 "mov r8 = cpuid[r8]\n");
366 DEFINE_FUNC1(get_pmd, int,
367 "mov r8 = pmd[r8]\n");
368 DEFINE_FUNC1(get_rr, unsigned long,
369 "mov r8 = rr[r8]\n");
370
371 DEFINE_VOID_FUNC0(ssm_i,
372 "ssm psr.i\n");
373 DEFINE_VOID_FUNC0(rsm_i,
374 "rsm psr.i\n");
375
376 extern void
377 ia64_native_set_rr0_to_rr4_func(unsigned long val0, unsigned long val1,
378 unsigned long val2, unsigned long val3,
379 unsigned long val4);
380 __DEFINE_FUNC(set_rr0_to_rr4,
381 "mov rr[r0] = r8\n"
382 "movl r2 = 0x2000000000000000\n"
383 ";;\n"
384 "mov rr[r2] = r9\n"
385 "shl r3 = r2, 1\n" /* movl r3 = 0x4000000000000000 */
386 ";;\n"
387 "add r2 = r2, r3\n" /* movl r2 = 0x6000000000000000 */
388 "mov rr[r3] = r10\n"
389 ";;\n"
390 "mov rr[r2] = r11\n"
391 "shl r3 = r3, 1\n" /* movl r3 = 0x8000000000000000 */
392 ";;\n"
393 "mov rr[r3] = r14\n");
394
395 extern unsigned long ia64_native_getreg_func(int regnum);
396 asm(".global ia64_native_getreg_func\n");
397 #define __DEFINE_GET_REG(id, reg) \
398 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
399 ";;\n" \
400 "cmp.eq p6, p0 = r2, r8\n" \
401 ";;\n" \
402 "(p6) mov r8 = " #reg "\n" \
403 "(p6) br.cond.sptk.many b6\n" \
404 ";;\n"
405 #define __DEFINE_GET_AR(id, reg) __DEFINE_GET_REG(AR_ ## id, ar.reg)
406 #define __DEFINE_GET_CR(id, reg) __DEFINE_GET_REG(CR_ ## id, cr.reg)
407
408 __DEFINE_FUNC(getreg,
409 __DEFINE_GET_REG(GP, gp)
410 /*__DEFINE_GET_REG(IP, ip)*/ /* returned ip value shouldn't be constant */
411 __DEFINE_GET_REG(PSR, psr)
412 __DEFINE_GET_REG(TP, tp)
413 __DEFINE_GET_REG(SP, sp)
414
415 __DEFINE_GET_REG(AR_KR0, ar0)
416 __DEFINE_GET_REG(AR_KR1, ar1)
417 __DEFINE_GET_REG(AR_KR2, ar2)
418 __DEFINE_GET_REG(AR_KR3, ar3)
419 __DEFINE_GET_REG(AR_KR4, ar4)
420 __DEFINE_GET_REG(AR_KR5, ar5)
421 __DEFINE_GET_REG(AR_KR6, ar6)
422 __DEFINE_GET_REG(AR_KR7, ar7)
423 __DEFINE_GET_AR(RSC, rsc)
424 __DEFINE_GET_AR(BSP, bsp)
425 __DEFINE_GET_AR(BSPSTORE, bspstore)
426 __DEFINE_GET_AR(RNAT, rnat)
427 __DEFINE_GET_AR(FCR, fcr)
428 __DEFINE_GET_AR(EFLAG, eflag)
429 __DEFINE_GET_AR(CSD, csd)
430 __DEFINE_GET_AR(SSD, ssd)
431 __DEFINE_GET_REG(AR_CFLAG, ar27)
432 __DEFINE_GET_AR(FSR, fsr)
433 __DEFINE_GET_AR(FIR, fir)
434 __DEFINE_GET_AR(FDR, fdr)
435 __DEFINE_GET_AR(CCV, ccv)
436 __DEFINE_GET_AR(UNAT, unat)
437 __DEFINE_GET_AR(FPSR, fpsr)
438 __DEFINE_GET_AR(ITC, itc)
439 __DEFINE_GET_AR(PFS, pfs)
440 __DEFINE_GET_AR(LC, lc)
441 __DEFINE_GET_AR(EC, ec)
442
443 __DEFINE_GET_CR(DCR, dcr)
444 __DEFINE_GET_CR(ITM, itm)
445 __DEFINE_GET_CR(IVA, iva)
446 __DEFINE_GET_CR(PTA, pta)
447 __DEFINE_GET_CR(IPSR, ipsr)
448 __DEFINE_GET_CR(ISR, isr)
449 __DEFINE_GET_CR(IIP, iip)
450 __DEFINE_GET_CR(IFA, ifa)
451 __DEFINE_GET_CR(ITIR, itir)
452 __DEFINE_GET_CR(IIPA, iipa)
453 __DEFINE_GET_CR(IFS, ifs)
454 __DEFINE_GET_CR(IIM, iim)
455 __DEFINE_GET_CR(IHA, iha)
456 __DEFINE_GET_CR(LID, lid)
457 __DEFINE_GET_CR(IVR, ivr)
458 __DEFINE_GET_CR(TPR, tpr)
459 __DEFINE_GET_CR(EOI, eoi)
460 __DEFINE_GET_CR(IRR0, irr0)
461 __DEFINE_GET_CR(IRR1, irr1)
462 __DEFINE_GET_CR(IRR2, irr2)
463 __DEFINE_GET_CR(IRR3, irr3)
464 __DEFINE_GET_CR(ITV, itv)
465 __DEFINE_GET_CR(PMV, pmv)
466 __DEFINE_GET_CR(CMCV, cmcv)
467 __DEFINE_GET_CR(LRR0, lrr0)
468 __DEFINE_GET_CR(LRR1, lrr1)
469
470 "mov r8 = -1\n" /* unsupported case */
471 );
472
473 extern void ia64_native_setreg_func(int regnum, unsigned long val);
474 asm(".global ia64_native_setreg_func\n");
475 #define __DEFINE_SET_REG(id, reg) \
476 "mov r2 = " __stringify(_IA64_REG_ ## id) "\n" \
477 ";;\n" \
478 "cmp.eq p6, p0 = r2, r9\n" \
479 ";;\n" \
480 "(p6) mov " #reg " = r8\n" \
481 "(p6) br.cond.sptk.many b6\n" \
482 ";;\n"
483 #define __DEFINE_SET_AR(id, reg) __DEFINE_SET_REG(AR_ ## id, ar.reg)
484 #define __DEFINE_SET_CR(id, reg) __DEFINE_SET_REG(CR_ ## id, cr.reg)
485 __DEFINE_FUNC(setreg,
486 "mov r2 = " __stringify(_IA64_REG_PSR_L) "\n"
487 ";;\n"
488 "cmp.eq p6, p0 = r2, r9\n"
489 ";;\n"
490 "(p6) mov psr.l = r8\n"
491 #ifdef HAVE_SERIALIZE_DIRECTIVE
492 ".serialize.data\n"
493 #endif
494 "(p6) br.cond.sptk.many b6\n"
495 __DEFINE_SET_REG(GP, gp)
496 __DEFINE_SET_REG(SP, sp)
497
498 __DEFINE_SET_REG(AR_KR0, ar0)
499 __DEFINE_SET_REG(AR_KR1, ar1)
500 __DEFINE_SET_REG(AR_KR2, ar2)
501 __DEFINE_SET_REG(AR_KR3, ar3)
502 __DEFINE_SET_REG(AR_KR4, ar4)
503 __DEFINE_SET_REG(AR_KR5, ar5)
504 __DEFINE_SET_REG(AR_KR6, ar6)
505 __DEFINE_SET_REG(AR_KR7, ar7)
506 __DEFINE_SET_AR(RSC, rsc)
507 __DEFINE_SET_AR(BSP, bsp)
508 __DEFINE_SET_AR(BSPSTORE, bspstore)
509 __DEFINE_SET_AR(RNAT, rnat)
510 __DEFINE_SET_AR(FCR, fcr)
511 __DEFINE_SET_AR(EFLAG, eflag)
512 __DEFINE_SET_AR(CSD, csd)
513 __DEFINE_SET_AR(SSD, ssd)
514 __DEFINE_SET_REG(AR_CFLAG, ar27)
515 __DEFINE_SET_AR(FSR, fsr)
516 __DEFINE_SET_AR(FIR, fir)
517 __DEFINE_SET_AR(FDR, fdr)
518 __DEFINE_SET_AR(CCV, ccv)
519 __DEFINE_SET_AR(UNAT, unat)
520 __DEFINE_SET_AR(FPSR, fpsr)
521 __DEFINE_SET_AR(ITC, itc)
522 __DEFINE_SET_AR(PFS, pfs)
523 __DEFINE_SET_AR(LC, lc)
524 __DEFINE_SET_AR(EC, ec)
525
526 __DEFINE_SET_CR(DCR, dcr)
527 __DEFINE_SET_CR(ITM, itm)
528 __DEFINE_SET_CR(IVA, iva)
529 __DEFINE_SET_CR(PTA, pta)
530 __DEFINE_SET_CR(IPSR, ipsr)
531 __DEFINE_SET_CR(ISR, isr)
532 __DEFINE_SET_CR(IIP, iip)
533 __DEFINE_SET_CR(IFA, ifa)
534 __DEFINE_SET_CR(ITIR, itir)
535 __DEFINE_SET_CR(IIPA, iipa)
536 __DEFINE_SET_CR(IFS, ifs)
537 __DEFINE_SET_CR(IIM, iim)
538 __DEFINE_SET_CR(IHA, iha)
539 __DEFINE_SET_CR(LID, lid)
540 __DEFINE_SET_CR(IVR, ivr)
541 __DEFINE_SET_CR(TPR, tpr)
542 __DEFINE_SET_CR(EOI, eoi)
543 __DEFINE_SET_CR(IRR0, irr0)
544 __DEFINE_SET_CR(IRR1, irr1)
545 __DEFINE_SET_CR(IRR2, irr2)
546 __DEFINE_SET_CR(IRR3, irr3)
547 __DEFINE_SET_CR(ITV, itv)
548 __DEFINE_SET_CR(PMV, pmv)
549 __DEFINE_SET_CR(CMCV, cmcv)
550 __DEFINE_SET_CR(LRR0, lrr0)
551 __DEFINE_SET_CR(LRR1, lrr1)
552 );
553 #endif
554
555 struct pv_cpu_ops pv_cpu_ops = {
556 .fc = ia64_native_fc_func,
557 .thash = ia64_native_thash_func,
558 .get_cpuid = ia64_native_get_cpuid_func,
559 .get_pmd = ia64_native_get_pmd_func,
560 .ptcga = ia64_native_ptcga_func,
561 .get_rr = ia64_native_get_rr_func,
562 .set_rr = ia64_native_set_rr_func,
563 .set_rr0_to_rr4 = ia64_native_set_rr0_to_rr4_func,
564 .ssm_i = ia64_native_ssm_i_func,
565 .getreg = ia64_native_getreg_func,
566 .setreg = ia64_native_setreg_func,
567 .rsm_i = ia64_native_rsm_i_func,
568 .get_psr_i = ia64_native_get_psr_i_func,
569 .intrin_local_irq_restore
570 = ia64_native_intrin_local_irq_restore_func,
571 };
572 EXPORT_SYMBOL(pv_cpu_ops);
573
574 /******************************************************************************
575 * replacement of hand written assembly codes.
576 */
577
578 void
paravirt_cpu_asm_init(const struct pv_cpu_asm_switch * cpu_asm_switch)579 paravirt_cpu_asm_init(const struct pv_cpu_asm_switch *cpu_asm_switch)
580 {
581 extern unsigned long paravirt_switch_to_targ;
582 extern unsigned long paravirt_leave_syscall_targ;
583 extern unsigned long paravirt_work_processed_syscall_targ;
584 extern unsigned long paravirt_leave_kernel_targ;
585
586 paravirt_switch_to_targ = cpu_asm_switch->switch_to;
587 paravirt_leave_syscall_targ = cpu_asm_switch->leave_syscall;
588 paravirt_work_processed_syscall_targ =
589 cpu_asm_switch->work_processed_syscall;
590 paravirt_leave_kernel_targ = cpu_asm_switch->leave_kernel;
591 }
592
593 /***************************************************************************
594 * pv_iosapic_ops
595 * iosapic read/write hooks.
596 */
597
598 static unsigned int
ia64_native_iosapic_read(char __iomem * iosapic,unsigned int reg)599 ia64_native_iosapic_read(char __iomem *iosapic, unsigned int reg)
600 {
601 return __ia64_native_iosapic_read(iosapic, reg);
602 }
603
604 static void
ia64_native_iosapic_write(char __iomem * iosapic,unsigned int reg,u32 val)605 ia64_native_iosapic_write(char __iomem *iosapic, unsigned int reg, u32 val)
606 {
607 __ia64_native_iosapic_write(iosapic, reg, val);
608 }
609
610 struct pv_iosapic_ops pv_iosapic_ops = {
611 .pcat_compat_init = ia64_native_iosapic_pcat_compat_init,
612 .__get_irq_chip = ia64_native_iosapic_get_irq_chip,
613
614 .__read = ia64_native_iosapic_read,
615 .__write = ia64_native_iosapic_write,
616 };
617
618 /***************************************************************************
619 * pv_irq_ops
620 * irq operations
621 */
622
623 struct pv_irq_ops pv_irq_ops = {
624 .register_ipi = ia64_native_register_ipi,
625
626 .assign_irq_vector = ia64_native_assign_irq_vector,
627 .free_irq_vector = ia64_native_free_irq_vector,
628 .register_percpu_irq = ia64_native_register_percpu_irq,
629
630 .resend_irq = ia64_native_resend_irq,
631 };
632
633 /***************************************************************************
634 * pv_time_ops
635 * time operations
636 */
637 struct static_key paravirt_steal_enabled;
638 struct static_key paravirt_steal_rq_enabled;
639
640 static int
ia64_native_do_steal_accounting(unsigned long * new_itm)641 ia64_native_do_steal_accounting(unsigned long *new_itm)
642 {
643 return 0;
644 }
645
646 struct pv_time_ops pv_time_ops = {
647 .do_steal_accounting = ia64_native_do_steal_accounting,
648 .sched_clock = ia64_native_sched_clock,
649 };
650
651 /***************************************************************************
652 * binary pacthing
653 * pv_init_ops.patch_bundle
654 */
655
656 #ifdef ASM_SUPPORTED
657 #define IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg) \
658 __DEFINE_FUNC(get_ ## name, \
659 ";;\n" \
660 "mov r8 = " #reg "\n" \
661 ";;\n")
662
663 #define IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
664 __DEFINE_FUNC(set_ ## name, \
665 ";;\n" \
666 "mov " #reg " = r8\n" \
667 ";;\n")
668
669 #define IA64_NATIVE_PATCH_DEFINE_REG(name, reg) \
670 IA64_NATIVE_PATCH_DEFINE_GET_REG(name, reg); \
671 IA64_NATIVE_PATCH_DEFINE_SET_REG(name, reg) \
672
673 #define IA64_NATIVE_PATCH_DEFINE_AR(name, reg) \
674 IA64_NATIVE_PATCH_DEFINE_REG(ar_ ## name, ar.reg)
675
676 #define IA64_NATIVE_PATCH_DEFINE_CR(name, reg) \
677 IA64_NATIVE_PATCH_DEFINE_REG(cr_ ## name, cr.reg)
678
679
680 IA64_NATIVE_PATCH_DEFINE_GET_REG(psr, psr);
681 IA64_NATIVE_PATCH_DEFINE_GET_REG(tp, tp);
682
683 /* IA64_NATIVE_PATCH_DEFINE_SET_REG(psr_l, psr.l); */
684 __DEFINE_FUNC(set_psr_l,
685 ";;\n"
686 "mov psr.l = r8\n"
687 #ifdef HAVE_SERIALIZE_DIRECTIVE
688 ".serialize.data\n"
689 #endif
690 ";;\n");
691
692 IA64_NATIVE_PATCH_DEFINE_REG(gp, gp);
693 IA64_NATIVE_PATCH_DEFINE_REG(sp, sp);
694
695 IA64_NATIVE_PATCH_DEFINE_REG(kr0, ar0);
696 IA64_NATIVE_PATCH_DEFINE_REG(kr1, ar1);
697 IA64_NATIVE_PATCH_DEFINE_REG(kr2, ar2);
698 IA64_NATIVE_PATCH_DEFINE_REG(kr3, ar3);
699 IA64_NATIVE_PATCH_DEFINE_REG(kr4, ar4);
700 IA64_NATIVE_PATCH_DEFINE_REG(kr5, ar5);
701 IA64_NATIVE_PATCH_DEFINE_REG(kr6, ar6);
702 IA64_NATIVE_PATCH_DEFINE_REG(kr7, ar7);
703
704 IA64_NATIVE_PATCH_DEFINE_AR(rsc, rsc);
705 IA64_NATIVE_PATCH_DEFINE_AR(bsp, bsp);
706 IA64_NATIVE_PATCH_DEFINE_AR(bspstore, bspstore);
707 IA64_NATIVE_PATCH_DEFINE_AR(rnat, rnat);
708 IA64_NATIVE_PATCH_DEFINE_AR(fcr, fcr);
709 IA64_NATIVE_PATCH_DEFINE_AR(eflag, eflag);
710 IA64_NATIVE_PATCH_DEFINE_AR(csd, csd);
711 IA64_NATIVE_PATCH_DEFINE_AR(ssd, ssd);
712 IA64_NATIVE_PATCH_DEFINE_REG(ar27, ar27);
713 IA64_NATIVE_PATCH_DEFINE_AR(fsr, fsr);
714 IA64_NATIVE_PATCH_DEFINE_AR(fir, fir);
715 IA64_NATIVE_PATCH_DEFINE_AR(fdr, fdr);
716 IA64_NATIVE_PATCH_DEFINE_AR(ccv, ccv);
717 IA64_NATIVE_PATCH_DEFINE_AR(unat, unat);
718 IA64_NATIVE_PATCH_DEFINE_AR(fpsr, fpsr);
719 IA64_NATIVE_PATCH_DEFINE_AR(itc, itc);
720 IA64_NATIVE_PATCH_DEFINE_AR(pfs, pfs);
721 IA64_NATIVE_PATCH_DEFINE_AR(lc, lc);
722 IA64_NATIVE_PATCH_DEFINE_AR(ec, ec);
723
724 IA64_NATIVE_PATCH_DEFINE_CR(dcr, dcr);
725 IA64_NATIVE_PATCH_DEFINE_CR(itm, itm);
726 IA64_NATIVE_PATCH_DEFINE_CR(iva, iva);
727 IA64_NATIVE_PATCH_DEFINE_CR(pta, pta);
728 IA64_NATIVE_PATCH_DEFINE_CR(ipsr, ipsr);
729 IA64_NATIVE_PATCH_DEFINE_CR(isr, isr);
730 IA64_NATIVE_PATCH_DEFINE_CR(iip, iip);
731 IA64_NATIVE_PATCH_DEFINE_CR(ifa, ifa);
732 IA64_NATIVE_PATCH_DEFINE_CR(itir, itir);
733 IA64_NATIVE_PATCH_DEFINE_CR(iipa, iipa);
734 IA64_NATIVE_PATCH_DEFINE_CR(ifs, ifs);
735 IA64_NATIVE_PATCH_DEFINE_CR(iim, iim);
736 IA64_NATIVE_PATCH_DEFINE_CR(iha, iha);
737 IA64_NATIVE_PATCH_DEFINE_CR(lid, lid);
738 IA64_NATIVE_PATCH_DEFINE_CR(ivr, ivr);
739 IA64_NATIVE_PATCH_DEFINE_CR(tpr, tpr);
740 IA64_NATIVE_PATCH_DEFINE_CR(eoi, eoi);
741 IA64_NATIVE_PATCH_DEFINE_CR(irr0, irr0);
742 IA64_NATIVE_PATCH_DEFINE_CR(irr1, irr1);
743 IA64_NATIVE_PATCH_DEFINE_CR(irr2, irr2);
744 IA64_NATIVE_PATCH_DEFINE_CR(irr3, irr3);
745 IA64_NATIVE_PATCH_DEFINE_CR(itv, itv);
746 IA64_NATIVE_PATCH_DEFINE_CR(pmv, pmv);
747 IA64_NATIVE_PATCH_DEFINE_CR(cmcv, cmcv);
748 IA64_NATIVE_PATCH_DEFINE_CR(lrr0, lrr0);
749 IA64_NATIVE_PATCH_DEFINE_CR(lrr1, lrr1);
750
751 static const struct paravirt_patch_bundle_elem ia64_native_patch_bundle_elems[]
752 __initdata_or_module =
753 {
754 #define IA64_NATIVE_PATCH_BUNDLE_ELEM(name, type) \
755 { \
756 (void*)ia64_native_ ## name ## _direct_start, \
757 (void*)ia64_native_ ## name ## _direct_end, \
758 PARAVIRT_PATCH_TYPE_ ## type, \
759 }
760
761 IA64_NATIVE_PATCH_BUNDLE_ELEM(fc, FC),
762 IA64_NATIVE_PATCH_BUNDLE_ELEM(thash, THASH),
763 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_cpuid, GET_CPUID),
764 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_pmd, GET_PMD),
765 IA64_NATIVE_PATCH_BUNDLE_ELEM(ptcga, PTCGA),
766 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_rr, GET_RR),
767 IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr, SET_RR),
768 IA64_NATIVE_PATCH_BUNDLE_ELEM(set_rr0_to_rr4, SET_RR0_TO_RR4),
769 IA64_NATIVE_PATCH_BUNDLE_ELEM(ssm_i, SSM_I),
770 IA64_NATIVE_PATCH_BUNDLE_ELEM(rsm_i, RSM_I),
771 IA64_NATIVE_PATCH_BUNDLE_ELEM(get_psr_i, GET_PSR_I),
772 IA64_NATIVE_PATCH_BUNDLE_ELEM(intrin_local_irq_restore,
773 INTRIN_LOCAL_IRQ_RESTORE),
774
775 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg) \
776 { \
777 (void*)ia64_native_get_ ## name ## _direct_start, \
778 (void*)ia64_native_get_ ## name ## _direct_end, \
779 PARAVIRT_PATCH_TYPE_GETREG + _IA64_REG_ ## reg, \
780 }
781
782 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
783 { \
784 (void*)ia64_native_set_ ## name ## _direct_start, \
785 (void*)ia64_native_set_ ## name ## _direct_end, \
786 PARAVIRT_PATCH_TYPE_SETREG + _IA64_REG_ ## reg, \
787 }
788
789 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(name, reg) \
790 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(name, reg), \
791 IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(name, reg) \
792
793 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(name, reg) \
794 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar_ ## name, AR_ ## reg)
795
796 #define IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(name, reg) \
797 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(cr_ ## name, CR_ ## reg)
798
799 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(psr, PSR),
800 IA64_NATIVE_PATCH_BUNDLE_ELEM_GETREG(tp, TP),
801
802 IA64_NATIVE_PATCH_BUNDLE_ELEM_SETREG(psr_l, PSR_L),
803
804 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(gp, GP),
805 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(sp, SP),
806
807 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr0, AR_KR0),
808 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr1, AR_KR1),
809 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr2, AR_KR2),
810 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr3, AR_KR3),
811 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr4, AR_KR4),
812 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr5, AR_KR5),
813 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr6, AR_KR6),
814 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(kr7, AR_KR7),
815
816 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rsc, RSC),
817 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bsp, BSP),
818 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(bspstore, BSPSTORE),
819 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(rnat, RNAT),
820 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fcr, FCR),
821 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(eflag, EFLAG),
822 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(csd, CSD),
823 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ssd, SSD),
824 IA64_NATIVE_PATCH_BUNDLE_ELEM_REG(ar27, AR_CFLAG),
825 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fsr, FSR),
826 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fir, FIR),
827 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fdr, FDR),
828 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ccv, CCV),
829 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(unat, UNAT),
830 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(fpsr, FPSR),
831 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(itc, ITC),
832 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(pfs, PFS),
833 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(lc, LC),
834 IA64_NATIVE_PATCH_BUNDLE_ELEM_AR(ec, EC),
835
836 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(dcr, DCR),
837 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itm, ITM),
838 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iva, IVA),
839 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pta, PTA),
840 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ipsr, IPSR),
841 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(isr, ISR),
842 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iip, IIP),
843 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifa, IFA),
844 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itir, ITIR),
845 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iipa, IIPA),
846 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ifs, IFS),
847 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iim, IIM),
848 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(iha, IHA),
849 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lid, LID),
850 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(ivr, IVR),
851 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(tpr, TPR),
852 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(eoi, EOI),
853 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr0, IRR0),
854 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr1, IRR1),
855 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr2, IRR2),
856 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(irr3, IRR3),
857 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(itv, ITV),
858 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(pmv, PMV),
859 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(cmcv, CMCV),
860 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr0, LRR0),
861 IA64_NATIVE_PATCH_BUNDLE_ELEM_CR(lrr1, LRR1),
862 };
863
864 unsigned long __init_or_module
ia64_native_patch_bundle(void * sbundle,void * ebundle,unsigned long type)865 ia64_native_patch_bundle(void *sbundle, void *ebundle, unsigned long type)
866 {
867 const unsigned long nelems = sizeof(ia64_native_patch_bundle_elems) /
868 sizeof(ia64_native_patch_bundle_elems[0]);
869
870 return __paravirt_patch_apply_bundle(sbundle, ebundle, type,
871 ia64_native_patch_bundle_elems,
872 nelems, NULL);
873 }
874 #endif /* ASM_SUPPOTED */
875
876 extern const char ia64_native_switch_to[];
877 extern const char ia64_native_leave_syscall[];
878 extern const char ia64_native_work_processed_syscall[];
879 extern const char ia64_native_leave_kernel[];
880
881 const struct paravirt_patch_branch_target ia64_native_branch_target[]
882 __initconst = {
883 #define PARAVIRT_BR_TARGET(name, type) \
884 { \
885 ia64_native_ ## name, \
886 PARAVIRT_PATCH_TYPE_BR_ ## type, \
887 }
888 PARAVIRT_BR_TARGET(switch_to, SWITCH_TO),
889 PARAVIRT_BR_TARGET(leave_syscall, LEAVE_SYSCALL),
890 PARAVIRT_BR_TARGET(work_processed_syscall, WORK_PROCESSED_SYSCALL),
891 PARAVIRT_BR_TARGET(leave_kernel, LEAVE_KERNEL),
892 };
893
894 static void __init
ia64_native_patch_branch(unsigned long tag,unsigned long type)895 ia64_native_patch_branch(unsigned long tag, unsigned long type)
896 {
897 const unsigned long nelem =
898 sizeof(ia64_native_branch_target) /
899 sizeof(ia64_native_branch_target[0]);
900 __paravirt_patch_apply_branch(tag, type,
901 ia64_native_branch_target, nelem);
902 }
903