1/* 2 * Compatibility mode system call entry point for x86-64. 3 * 4 * Copyright 2000-2002 Andi Kleen, SuSE Labs. 5 */ 6 7#include <asm/dwarf2.h> 8#include <asm/calling.h> 9#include <asm/asm-offsets.h> 10#include <asm/current.h> 11#include <asm/errno.h> 12#include <asm/ia32_unistd.h> 13#include <asm/thread_info.h> 14#include <asm/segment.h> 15#include <asm/irqflags.h> 16#include <linux/linkage.h> 17#include <linux/err.h> 18 19/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 20#include <linux/elf-em.h> 21#define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE) 22#define __AUDIT_ARCH_LE 0x40000000 23 24#ifndef CONFIG_AUDITSYSCALL 25#define sysexit_audit ia32_ret_from_sys_call 26#define sysretl_audit ia32_ret_from_sys_call 27#endif 28 29 .section .entry.text, "ax" 30 31 .macro IA32_ARG_FIXUP noebp=0 32 movl %edi,%r8d 33 .if \noebp 34 .else 35 movl %ebp,%r9d 36 .endif 37 xchg %ecx,%esi 38 movl %ebx,%edi 39 movl %edx,%edx /* zero extension */ 40 .endm 41 42 /* clobbers %eax */ 43 .macro CLEAR_RREGS offset=0, _r9=rax 44 xorl %eax,%eax 45 movq %rax,\offset+R11(%rsp) 46 movq %rax,\offset+R10(%rsp) 47 movq %\_r9,\offset+R9(%rsp) 48 movq %rax,\offset+R8(%rsp) 49 .endm 50 51 /* 52 * Reload arg registers from stack in case ptrace changed them. 53 * We don't reload %eax because syscall_trace_enter() returned 54 * the %rax value we should see. Instead, we just truncate that 55 * value to 32 bits again as we did on entry from user mode. 56 * If it's a new value set by user_regset during entry tracing, 57 * this matches the normal truncation of the user-mode value. 58 * If it's -1 to make us punt the syscall, then (u32)-1 is still 59 * an appropriately invalid value. 60 */ 61 .macro LOAD_ARGS32 offset, _r9=0 62 .if \_r9 63 movl \offset+16(%rsp),%r9d 64 .endif 65 movl \offset+40(%rsp),%ecx 66 movl \offset+48(%rsp),%edx 67 movl \offset+56(%rsp),%esi 68 movl \offset+64(%rsp),%edi 69 movl %eax,%eax /* zero extension */ 70 .endm 71 72 .macro CFI_STARTPROC32 simple 73 CFI_STARTPROC \simple 74 CFI_UNDEFINED r8 75 CFI_UNDEFINED r9 76 CFI_UNDEFINED r10 77 CFI_UNDEFINED r11 78 CFI_UNDEFINED r12 79 CFI_UNDEFINED r13 80 CFI_UNDEFINED r14 81 CFI_UNDEFINED r15 82 .endm 83 84#ifdef CONFIG_PARAVIRT 85ENTRY(native_usergs_sysret32) 86 swapgs 87 sysretl 88ENDPROC(native_usergs_sysret32) 89 90ENTRY(native_irq_enable_sysexit) 91 swapgs 92 sti 93 sysexit 94ENDPROC(native_irq_enable_sysexit) 95#endif 96 97/* 98 * 32bit SYSENTER instruction entry. 99 * 100 * Arguments: 101 * %eax System call number. 102 * %ebx Arg1 103 * %ecx Arg2 104 * %edx Arg3 105 * %esi Arg4 106 * %edi Arg5 107 * %ebp user stack 108 * 0(%ebp) Arg6 109 * 110 * Interrupts off. 111 * 112 * This is purely a fast path. For anything complicated we use the int 0x80 113 * path below. Set up a complete hardware stack frame to share code 114 * with the int 0x80 path. 115 */ 116ENTRY(ia32_sysenter_target) 117 CFI_STARTPROC32 simple 118 CFI_SIGNAL_FRAME 119 CFI_DEF_CFA rsp,0 120 CFI_REGISTER rsp,rbp 121 SWAPGS_UNSAFE_STACK 122 movq PER_CPU_VAR(kernel_stack), %rsp 123 addq $(KERNEL_STACK_OFFSET),%rsp 124 /* 125 * No need to follow this irqs on/off section: the syscall 126 * disabled irqs, here we enable it straight after entry: 127 */ 128 ENABLE_INTERRUPTS(CLBR_NONE) 129 movl %ebp,%ebp /* zero extension */ 130 pushq_cfi $__USER32_DS 131 /*CFI_REL_OFFSET ss,0*/ 132 pushq_cfi %rbp 133 CFI_REL_OFFSET rsp,0 134 pushfq_cfi 135 /*CFI_REL_OFFSET rflags,0*/ 136 movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d 137 CFI_REGISTER rip,r10 138 pushq_cfi $__USER32_CS 139 /*CFI_REL_OFFSET cs,0*/ 140 movl %eax, %eax 141 pushq_cfi %r10 142 CFI_REL_OFFSET rip,0 143 pushq_cfi %rax 144 cld 145 SAVE_ARGS 0,1,0 146 /* no need to do an access_ok check here because rbp has been 147 32bit zero extended */ 1481: movl (%rbp),%ebp 149 .section __ex_table,"a" 150 .quad 1b,ia32_badarg 151 .previous 152 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 153 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 154 CFI_REMEMBER_STATE 155 jnz sysenter_tracesys 156 cmpq $(IA32_NR_syscalls-1),%rax 157 ja ia32_badsys 158sysenter_do_call: 159 IA32_ARG_FIXUP 160sysenter_dispatch: 161 call *ia32_sys_call_table(,%rax,8) 162 movq %rax,RAX-ARGOFFSET(%rsp) 163 DISABLE_INTERRUPTS(CLBR_NONE) 164 TRACE_IRQS_OFF 165 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 166 jnz sysexit_audit 167sysexit_from_sys_call: 168 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 169 /* clear IF, that popfq doesn't enable interrupts early */ 170 andl $~0x200,EFLAGS-R11(%rsp) 171 movl RIP-R11(%rsp),%edx /* User %eip */ 172 CFI_REGISTER rip,rdx 173 RESTORE_ARGS 0,24,0,0,0,0 174 xorq %r8,%r8 175 xorq %r9,%r9 176 xorq %r10,%r10 177 xorq %r11,%r11 178 popfq_cfi 179 /*CFI_RESTORE rflags*/ 180 popq_cfi %rcx /* User %esp */ 181 CFI_REGISTER rsp,rcx 182 TRACE_IRQS_ON 183 ENABLE_INTERRUPTS_SYSEXIT32 184 185#ifdef CONFIG_AUDITSYSCALL 186 .macro auditsys_entry_common 187 movl %esi,%r9d /* 6th arg: 4th syscall arg */ 188 movl %edx,%r8d /* 5th arg: 3rd syscall arg */ 189 /* (already in %ecx) 4th arg: 2nd syscall arg */ 190 movl %ebx,%edx /* 3rd arg: 1st syscall arg */ 191 movl %eax,%esi /* 2nd arg: syscall number */ 192 movl $AUDIT_ARCH_I386,%edi /* 1st arg: audit arch */ 193 call __audit_syscall_entry 194 movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */ 195 cmpq $(IA32_NR_syscalls-1),%rax 196 ja ia32_badsys 197 movl %ebx,%edi /* reload 1st syscall arg */ 198 movl RCX-ARGOFFSET(%rsp),%esi /* reload 2nd syscall arg */ 199 movl RDX-ARGOFFSET(%rsp),%edx /* reload 3rd syscall arg */ 200 movl RSI-ARGOFFSET(%rsp),%ecx /* reload 4th syscall arg */ 201 movl RDI-ARGOFFSET(%rsp),%r8d /* reload 5th syscall arg */ 202 .endm 203 204 .macro auditsys_exit exit 205 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 206 jnz ia32_ret_from_sys_call 207 TRACE_IRQS_ON 208 ENABLE_INTERRUPTS(CLBR_NONE) 209 movl %eax,%esi /* second arg, syscall return value */ 210 cmpl $-MAX_ERRNO,%eax /* is it an error ? */ 211 jbe 1f 212 movslq %eax, %rsi /* if error sign extend to 64 bits */ 2131: setbe %al /* 1 if error, 0 if not */ 214 movzbl %al,%edi /* zero-extend that into %edi */ 215 call __audit_syscall_exit 216 movq RAX-ARGOFFSET(%rsp),%rax /* reload syscall return value */ 217 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi 218 DISABLE_INTERRUPTS(CLBR_NONE) 219 TRACE_IRQS_OFF 220 testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 221 jz \exit 222 CLEAR_RREGS -ARGOFFSET 223 jmp int_with_check 224 .endm 225 226sysenter_auditsys: 227 CFI_RESTORE_STATE 228 auditsys_entry_common 229 movl %ebp,%r9d /* reload 6th syscall arg */ 230 jmp sysenter_dispatch 231 232sysexit_audit: 233 auditsys_exit sysexit_from_sys_call 234#endif 235 236sysenter_tracesys: 237#ifdef CONFIG_AUDITSYSCALL 238 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 239 jz sysenter_auditsys 240#endif 241 SAVE_REST 242 CLEAR_RREGS 243 movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */ 244 movq %rsp,%rdi /* &pt_regs -> arg1 */ 245 call syscall_trace_enter 246 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */ 247 RESTORE_REST 248 cmpq $(IA32_NR_syscalls-1),%rax 249 ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */ 250 jmp sysenter_do_call 251 CFI_ENDPROC 252ENDPROC(ia32_sysenter_target) 253 254/* 255 * 32bit SYSCALL instruction entry. 256 * 257 * Arguments: 258 * %eax System call number. 259 * %ebx Arg1 260 * %ecx return EIP 261 * %edx Arg3 262 * %esi Arg4 263 * %edi Arg5 264 * %ebp Arg2 [note: not saved in the stack frame, should not be touched] 265 * %esp user stack 266 * 0(%esp) Arg6 267 * 268 * Interrupts off. 269 * 270 * This is purely a fast path. For anything complicated we use the int 0x80 271 * path below. Set up a complete hardware stack frame to share code 272 * with the int 0x80 path. 273 */ 274ENTRY(ia32_cstar_target) 275 CFI_STARTPROC32 simple 276 CFI_SIGNAL_FRAME 277 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET 278 CFI_REGISTER rip,rcx 279 /*CFI_REGISTER rflags,r11*/ 280 SWAPGS_UNSAFE_STACK 281 movl %esp,%r8d 282 CFI_REGISTER rsp,r8 283 movq PER_CPU_VAR(kernel_stack),%rsp 284 /* 285 * No need to follow this irqs on/off section: the syscall 286 * disabled irqs and here we enable it straight after entry: 287 */ 288 ENABLE_INTERRUPTS(CLBR_NONE) 289 SAVE_ARGS 8,0,0 290 movl %eax,%eax /* zero extension */ 291 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 292 movq %rcx,RIP-ARGOFFSET(%rsp) 293 CFI_REL_OFFSET rip,RIP-ARGOFFSET 294 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */ 295 movl %ebp,%ecx 296 movq $__USER32_CS,CS-ARGOFFSET(%rsp) 297 movq $__USER32_DS,SS-ARGOFFSET(%rsp) 298 movq %r11,EFLAGS-ARGOFFSET(%rsp) 299 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/ 300 movq %r8,RSP-ARGOFFSET(%rsp) 301 CFI_REL_OFFSET rsp,RSP-ARGOFFSET 302 /* no need to do an access_ok check here because r8 has been 303 32bit zero extended */ 304 /* hardware stack frame is complete now */ 3051: movl (%r8),%r9d 306 .section __ex_table,"a" 307 .quad 1b,ia32_badarg 308 .previous 309 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 310 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 311 CFI_REMEMBER_STATE 312 jnz cstar_tracesys 313 cmpq $IA32_NR_syscalls-1,%rax 314 ja ia32_badsys 315cstar_do_call: 316 IA32_ARG_FIXUP 1 317cstar_dispatch: 318 call *ia32_sys_call_table(,%rax,8) 319 movq %rax,RAX-ARGOFFSET(%rsp) 320 DISABLE_INTERRUPTS(CLBR_NONE) 321 TRACE_IRQS_OFF 322 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 323 jnz sysretl_audit 324sysretl_from_sys_call: 325 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 326 RESTORE_ARGS 0,-ARG_SKIP,0,0,0 327 movl RIP-ARGOFFSET(%rsp),%ecx 328 CFI_REGISTER rip,rcx 329 movl EFLAGS-ARGOFFSET(%rsp),%r11d 330 /*CFI_REGISTER rflags,r11*/ 331 xorq %r10,%r10 332 xorq %r9,%r9 333 xorq %r8,%r8 334 TRACE_IRQS_ON 335 movl RSP-ARGOFFSET(%rsp),%esp 336 CFI_RESTORE rsp 337 USERGS_SYSRET32 338 339#ifdef CONFIG_AUDITSYSCALL 340cstar_auditsys: 341 CFI_RESTORE_STATE 342 movl %r9d,R9-ARGOFFSET(%rsp) /* register to be clobbered by call */ 343 auditsys_entry_common 344 movl R9-ARGOFFSET(%rsp),%r9d /* reload 6th syscall arg */ 345 jmp cstar_dispatch 346 347sysretl_audit: 348 auditsys_exit sysretl_from_sys_call 349#endif 350 351cstar_tracesys: 352#ifdef CONFIG_AUDITSYSCALL 353 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 354 jz cstar_auditsys 355#endif 356 xchgl %r9d,%ebp 357 SAVE_REST 358 CLEAR_RREGS 0, r9 359 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ 360 movq %rsp,%rdi /* &pt_regs -> arg1 */ 361 call syscall_trace_enter 362 LOAD_ARGS32 ARGOFFSET, 1 /* reload args from stack in case ptrace changed it */ 363 RESTORE_REST 364 xchgl %ebp,%r9d 365 cmpq $(IA32_NR_syscalls-1),%rax 366 ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */ 367 jmp cstar_do_call 368END(ia32_cstar_target) 369 370ia32_badarg: 371 movq $-EFAULT,%rax 372 jmp ia32_sysret 373 CFI_ENDPROC 374 375/* 376 * Emulated IA32 system calls via int 0x80. 377 * 378 * Arguments: 379 * %eax System call number. 380 * %ebx Arg1 381 * %ecx Arg2 382 * %edx Arg3 383 * %esi Arg4 384 * %edi Arg5 385 * %ebp Arg6 [note: not saved in the stack frame, should not be touched] 386 * 387 * Notes: 388 * Uses the same stack frame as the x86-64 version. 389 * All registers except %eax must be saved (but ptrace may violate that) 390 * Arguments are zero extended. For system calls that want sign extension and 391 * take long arguments a wrapper is needed. Most calls can just be called 392 * directly. 393 * Assumes it is only called from user space and entered with interrupts off. 394 */ 395 396ENTRY(ia32_syscall) 397 CFI_STARTPROC32 simple 398 CFI_SIGNAL_FRAME 399 CFI_DEF_CFA rsp,SS+8-RIP 400 /*CFI_REL_OFFSET ss,SS-RIP*/ 401 CFI_REL_OFFSET rsp,RSP-RIP 402 /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/ 403 /*CFI_REL_OFFSET cs,CS-RIP*/ 404 CFI_REL_OFFSET rip,RIP-RIP 405 PARAVIRT_ADJUST_EXCEPTION_FRAME 406 SWAPGS 407 /* 408 * No need to follow this irqs on/off section: the syscall 409 * disabled irqs and here we enable it straight after entry: 410 */ 411 ENABLE_INTERRUPTS(CLBR_NONE) 412 movl %eax,%eax 413 pushq_cfi %rax 414 cld 415 /* note the registers are not zero extended to the sf. 416 this could be a problem. */ 417 SAVE_ARGS 0,1,0 418 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 419 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 420 jnz ia32_tracesys 421 cmpq $(IA32_NR_syscalls-1),%rax 422 ja ia32_badsys 423ia32_do_call: 424 IA32_ARG_FIXUP 425 call *ia32_sys_call_table(,%rax,8) # xxx: rip relative 426ia32_sysret: 427 movq %rax,RAX-ARGOFFSET(%rsp) 428ia32_ret_from_sys_call: 429 CLEAR_RREGS -ARGOFFSET 430 jmp int_ret_from_sys_call 431 432ia32_tracesys: 433 SAVE_REST 434 CLEAR_RREGS 435 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */ 436 movq %rsp,%rdi /* &pt_regs -> arg1 */ 437 call syscall_trace_enter 438 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */ 439 RESTORE_REST 440 cmpq $(IA32_NR_syscalls-1),%rax 441 ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */ 442 jmp ia32_do_call 443END(ia32_syscall) 444 445ia32_badsys: 446 movq $0,ORIG_RAX-ARGOFFSET(%rsp) 447 movq $-ENOSYS,%rax 448 jmp ia32_sysret 449 450 CFI_ENDPROC 451 452 .macro PTREGSCALL label, func, arg 453 ALIGN 454GLOBAL(\label) 455 leaq \func(%rip),%rax 456 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */ 457 jmp ia32_ptregs_common 458 .endm 459 460 CFI_STARTPROC32 461 462 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi 463 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi 464 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx 465 PTREGSCALL stub32_execve, sys32_execve, %rcx 466 PTREGSCALL stub32_fork, sys_fork, %rdi 467 PTREGSCALL stub32_clone, sys32_clone, %rdx 468 PTREGSCALL stub32_vfork, sys_vfork, %rdi 469 PTREGSCALL stub32_iopl, sys_iopl, %rsi 470 471 ALIGN 472ia32_ptregs_common: 473 popq %r11 474 CFI_ENDPROC 475 CFI_STARTPROC32 simple 476 CFI_SIGNAL_FRAME 477 CFI_DEF_CFA rsp,SS+8-ARGOFFSET 478 CFI_REL_OFFSET rax,RAX-ARGOFFSET 479 CFI_REL_OFFSET rcx,RCX-ARGOFFSET 480 CFI_REL_OFFSET rdx,RDX-ARGOFFSET 481 CFI_REL_OFFSET rsi,RSI-ARGOFFSET 482 CFI_REL_OFFSET rdi,RDI-ARGOFFSET 483 CFI_REL_OFFSET rip,RIP-ARGOFFSET 484/* CFI_REL_OFFSET cs,CS-ARGOFFSET*/ 485/* CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/ 486 CFI_REL_OFFSET rsp,RSP-ARGOFFSET 487/* CFI_REL_OFFSET ss,SS-ARGOFFSET*/ 488 SAVE_REST 489 call *%rax 490 RESTORE_REST 491 jmp ia32_sysret /* misbalances the return cache */ 492 CFI_ENDPROC 493END(ia32_ptregs_common) 494