Commit 50d7280d authored by Martin Schwidefsky's avatar Martin Schwidefsky Committed by Martin Schwidefsky
Browse files

[S390] fault handler performance optimization.



Slim down the do_exception function to handle only the fast path of a
fault and move the exceptional cases into a new function. That slightly
increases the performance of the fault handling.

Build fix for !CONFIG_COMPAT by
Kamalesh Babulal <kamalesh@linux.vnet.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 7ecb344a
......@@ -34,16 +34,15 @@
#include <asm/pgtable.h>
#include <asm/s390_ext.h>
#include <asm/mmu_context.h>
#include <asm/compat.h>
#include "../kernel/entry.h"
#ifndef CONFIG_64BIT
#define __FAIL_ADDR_MASK 0x7ffff000
#define __FIXUP_MASK 0x7fffffff
#define __SUBCODE_MASK 0x0200
#define __PF_RES_FIELD 0ULL
#else /* CONFIG_64BIT */
#define __FAIL_ADDR_MASK -4096L
#define __FIXUP_MASK ~0L
#define __SUBCODE_MASK 0x0600
#define __PF_RES_FIELD 0x8000000000000000ULL
#endif /* CONFIG_64BIT */
......@@ -52,6 +51,10 @@
extern int sysctl_userprocess_debug;
#endif
#define VM_FAULT_BADCONTEXT 0x010000
#define VM_FAULT_BADMAP 0x020000
#define VM_FAULT_BADACCESS 0x040000
static inline int notify_page_fault(struct pt_regs *regs)
{
int ret = 0;
......@@ -122,18 +125,22 @@ static inline int user_space_fault(unsigned long trans_exc_code)
* Send SIGSEGV to task. This is an external routine
* to keep the stack usage of do_page_fault small.
*/
static void do_sigsegv(struct pt_regs *regs, unsigned long error_code,
int si_code, unsigned long address)
static noinline void do_sigsegv(struct pt_regs *regs, long int_code,
int si_code, unsigned long trans_exc_code)
{
struct siginfo si;
unsigned long address;
address = trans_exc_code & __FAIL_ADDR_MASK;
current->thread.prot_addr = address;
current->thread.trap_no = int_code;
#if defined(CONFIG_SYSCTL) || defined(CONFIG_PROCESS_DEBUG)
#if defined(CONFIG_SYSCTL)
if (sysctl_userprocess_debug)
#endif
{
printk("User process fault: interruption code 0x%lX\n",
error_code);
int_code);
printk("failing address: %lX\n", address);
show_regs(regs);
}
......@@ -144,14 +151,14 @@ static void do_sigsegv(struct pt_regs *regs, unsigned long error_code,
force_sig_info(SIGSEGV, &si, current);
}
static void do_no_context(struct pt_regs *regs, unsigned long error_code,
unsigned long trans_exc_code)
static noinline void do_no_context(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code)
{
const struct exception_table_entry *fixup;
unsigned long address;
/* Are we prepared to handle this kernel fault? */
fixup = search_exception_tables(regs->psw.addr & __FIXUP_MASK);
fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
if (fixup) {
regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
return;
......@@ -169,107 +176,127 @@ static void do_no_context(struct pt_regs *regs, unsigned long error_code,
printk(KERN_ALERT "Unable to handle kernel paging request"
" at virtual user address %p\n", (void *)address);
die("Oops", regs, error_code);
die("Oops", regs, int_code);
do_exit(SIGKILL);
}
static void do_low_address(struct pt_regs *regs, unsigned long error_code,
unsigned long trans_exc_code)
static noinline void do_low_address(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code)
{
/* Low-address protection hit in kernel mode means
NULL pointer write access in kernel mode. */
if (regs->psw.mask & PSW_MASK_PSTATE) {
/* Low-address protection hit in user mode 'cannot happen'. */
die ("Low-address protection", regs, error_code);
die ("Low-address protection", regs, int_code);
do_exit(SIGKILL);
}
do_no_context(regs, error_code, trans_exc_code);
do_no_context(regs, int_code, trans_exc_code);
}
static void do_sigbus(struct pt_regs *regs, unsigned long error_code,
unsigned long trans_exc_code)
static noinline void do_sigbus(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code)
{
struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
up_read(&mm->mmap_sem);
/*
* Send a sigbus, regardless of whether we were in kernel
* or user mode.
*/
tsk->thread.prot_addr = trans_exc_code & __FAIL_ADDR_MASK;
tsk->thread.trap_no = error_code;
tsk->thread.trap_no = int_code;
force_sig(SIGBUS, tsk);
/* Kernel mode? Handle exceptions or die */
if (!(regs->psw.mask & PSW_MASK_PSTATE))
do_no_context(regs, error_code, trans_exc_code);
}
#ifdef CONFIG_S390_EXEC_PROTECT
static int signal_return(struct mm_struct *mm, struct pt_regs *regs,
unsigned long address, unsigned long error_code)
static noinline int signal_return(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code)
{
u16 instruction;
int rc;
#ifdef CONFIG_COMPAT
int compat;
#endif
pagefault_disable();
rc = __get_user(instruction, (u16 __user *) regs->psw.addr);
pagefault_enable();
if (rc)
return -EFAULT;
up_read(&mm->mmap_sem);
clear_tsk_thread_flag(current, TIF_SINGLE_STEP);
#ifdef CONFIG_COMPAT
compat = is_compat_task();
if (compat && instruction == 0x0a77)
sys32_sigreturn();
else if (compat && instruction == 0x0aad)
sys32_rt_sigreturn();
else
#endif
if (instruction == 0x0a77)
sys_sigreturn();
else if (instruction == 0x0aad)
sys_rt_sigreturn();
else {
current->thread.prot_addr = address;
current->thread.trap_no = error_code;
do_sigsegv(regs, error_code, SEGV_MAPERR, address);
}
if (!rc && instruction == 0x0a77) {
clear_tsk_thread_flag(current, TIF_SINGLE_STEP);
if (is_compat_task())
sys32_sigreturn();
else
sys_sigreturn();
} else if (!rc && instruction == 0x0aad) {
clear_tsk_thread_flag(current, TIF_SINGLE_STEP);
if (is_compat_task())
sys32_rt_sigreturn();
else
sys_rt_sigreturn();
} else
do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code);
return 0;
}
#endif /* CONFIG_S390_EXEC_PROTECT */
static noinline void do_fault_error(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code, int fault)
{
int si_code;
switch (fault) {
case VM_FAULT_BADACCESS:
#ifdef CONFIG_S390_EXEC_PROTECT
if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
(trans_exc_code & 3) == 0) {
signal_return(regs, int_code, trans_exc_code);
break;
}
#endif /* CONFIG_S390_EXEC_PROTECT */
case VM_FAULT_BADMAP:
/* Bad memory access. Check if it is kernel or user space. */
if (regs->psw.mask & PSW_MASK_PSTATE) {
/* User mode accesses just cause a SIGSEGV */
si_code = (fault == VM_FAULT_BADMAP) ?
SEGV_MAPERR : SEGV_ACCERR;
do_sigsegv(regs, int_code, si_code, trans_exc_code);
return;
}
case VM_FAULT_BADCONTEXT:
do_no_context(regs, int_code, trans_exc_code);
break;
default: /* fault & VM_FAULT_ERROR */
if (fault & VM_FAULT_OOM)
pagefault_out_of_memory();
else if (fault & VM_FAULT_SIGBUS) {
do_sigbus(regs, int_code, trans_exc_code);
/* Kernel mode? Handle exceptions or die */
if (!(regs->psw.mask & PSW_MASK_PSTATE))
do_no_context(regs, int_code, trans_exc_code);
} else
BUG();
break;
}
}
/*
* This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate
* routines.
*
* error_code:
* interruption code (int_code):
* 04 Protection -> Write-Protection (suprression)
* 10 Segment translation -> Not present (nullification)
* 11 Page translation -> Not present (nullification)
* 3b Region third trans. -> Not present (nullification)
*/
static inline void
do_exception(struct pt_regs *regs, unsigned long error_code, int write,
unsigned long trans_exc_code)
static inline int do_exception(struct pt_regs *regs, int write,
unsigned long trans_exc_code)
{
struct task_struct *tsk;
struct mm_struct *mm;
struct vm_area_struct *vma;
unsigned long address;
int si_code;
int fault;
if (notify_page_fault(regs))
return;
return 0;
tsk = current;
mm = tsk->mm;
......@@ -279,8 +306,9 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
* we are not in an interrupt and that there is a
* user context.
*/
fault = VM_FAULT_BADCONTEXT;
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
goto no_context;
goto out;
address = trans_exc_code & __FAIL_ADDR_MASK;
/*
......@@ -292,41 +320,35 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
down_read(&mm->mmap_sem);
si_code = SEGV_MAPERR;
fault = VM_FAULT_BADMAP;
vma = find_vma(mm, address);
if (!vma)
goto bad_area;
goto out_up;
if (unlikely(vma->vm_start > address)) {
if (!(vma->vm_flags & VM_GROWSDOWN))
goto out_up;
if (expand_stack(vma, address))
goto out_up;
}
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
fault = VM_FAULT_BADACCESS;
#ifdef CONFIG_S390_EXEC_PROTECT
if (unlikely((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
(trans_exc_code & 3) == 0 && !(vma->vm_flags & VM_EXEC)))
if (!signal_return(mm, regs, address, error_code))
/*
* signal_return() has done an up_read(&mm->mmap_sem)
* if it returns 0.
*/
return;
goto out_up;
#endif
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
si_code = SEGV_ACCERR;
if (!write) {
/* page not present, check vm flags */
if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
goto bad_area;
goto out_up;
} else {
if (!(vma->vm_flags & VM_WRITE))
goto bad_area;
goto out_up;
}
if (is_vm_hugetlb_page(vma))
......@@ -337,17 +359,9 @@ good_area:
* the fault.
*/
fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0);
if (unlikely(fault & VM_FAULT_ERROR)) {
if (fault & VM_FAULT_OOM) {
up_read(&mm->mmap_sem);
pagefault_out_of_memory();
return;
} else if (fault & VM_FAULT_SIGBUS) {
do_sigbus(regs, error_code, address);
return;
}
BUG();
}
if (unlikely(fault & VM_FAULT_ERROR))
goto out_up;
if (fault & VM_FAULT_MAJOR) {
tsk->maj_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
......@@ -357,67 +371,55 @@ good_area:
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
regs, address);
}
up_read(&mm->mmap_sem);
/*
* The instruction that caused the program check will
* be repeated. Don't signal single step via SIGTRAP.
*/
clear_tsk_thread_flag(tsk, TIF_SINGLE_STEP);
return;
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
fault = 0;
out_up:
up_read(&mm->mmap_sem);
/* User mode accesses just cause a SIGSEGV */
if (regs->psw.mask & PSW_MASK_PSTATE) {
tsk->thread.prot_addr = address;
tsk->thread.trap_no = error_code;
do_sigsegv(regs, error_code, si_code, address);
return;
}
no_context:
do_no_context(regs, error_code, trans_exc_code);
out:
return fault;
}
void __kprobes do_protection_exception(struct pt_regs *regs,
long error_code)
void __kprobes do_protection_exception(struct pt_regs *regs, long int_code)
{
unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
int fault;
/* Protection exception is supressing, decrement psw address. */
regs->psw.addr -= (error_code >> 16);
regs->psw.addr -= (int_code >> 16);
/*
* Check for low-address protection. This needs to be treated
* as a special case because the translation exception code
* field is not guaranteed to contain valid data in this case.
*/
if (unlikely(!(trans_exc_code & 4))) {
do_low_address(regs, error_code, trans_exc_code);
do_low_address(regs, int_code, trans_exc_code);
return;
}
do_exception(regs, 4, 1, trans_exc_code);
fault = do_exception(regs, 1, trans_exc_code);
if (unlikely(fault))
do_fault_error(regs, 4, trans_exc_code, fault);
}
void __kprobes do_dat_exception(struct pt_regs *regs, long error_code)
void __kprobes do_dat_exception(struct pt_regs *regs, long int_code)
{
do_exception(regs, error_code & 0xff, 0, S390_lowcore.trans_exc_code);
unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
int fault;
fault = do_exception(regs, 0, trans_exc_code);
if (unlikely(fault))
do_fault_error(regs, int_code & 255, trans_exc_code, fault);
}
#ifdef CONFIG_64BIT
void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code)
void __kprobes do_asce_exception(struct pt_regs *regs, long int_code)
{
unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
struct mm_struct *mm;
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
unsigned long address;
mm = current->mm;
address = trans_exc_code & __FAIL_ADDR_MASK;
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
goto no_context;
......@@ -425,7 +427,7 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code)
local_irq_enable();
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK);
up_read(&mm->mmap_sem);
if (vma) {
......@@ -435,14 +437,12 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code)
/* User mode accesses just cause a SIGSEGV */
if (regs->psw.mask & PSW_MASK_PSTATE) {
current->thread.prot_addr = address;
current->thread.trap_no = error_code;
do_sigsegv(regs, error_code, SEGV_MAPERR, address);
do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code);
return;
}
no_context:
do_no_context(regs, error_code, trans_exc_code);
do_no_context(regs, int_code, trans_exc_code);
}
#endif
......@@ -507,7 +507,7 @@ void pfault_fini(void)
: : "a" (&refbk), "m" (refbk) : "cc");
}
static void pfault_interrupt(__u16 error_code)
static void pfault_interrupt(__u16 int_code)
{
struct task_struct *tsk;
__u16 subcode;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment