From cbc9586c133420e7ed80bd87049c65e74026d8fe Mon Sep 17 00:00:00 2001 From: Tomas Hruby Date: Mon, 7 Jun 2010 07:43:17 +0000 Subject: [PATCH] Lazy FPU - FPU context is stored only if conflict between 2 FPU users or while exporting context of a process to userspace while it is the active user of FPU - FPU has its owner (fpu_owner) which points to the process whose state is currently loaded in FPU - the FPU exception is only turned on when scheduling a process which is not the owner of FPU - FPU state is restored for the process that generated the FPU exception. This process runs immediately without letting scheduler to pick a new process to resolve the FPU conflict asap, to minimize the FPU thrashing and FPU exception hadler execution - faster all non-FPU-exception kernel entries as FPU state is not checked nor saved - removed MF_USED_FPU flag, only MF_FPU_INITIALIZED remains to signal that a process has used FPU in the past --- kernel/arch/i386/arch_system.c | 26 +----------------------- kernel/arch/i386/exception.c | 12 ++++++++++++ kernel/arch/i386/mpx.S | 9 +++------ kernel/arch/i386/sconst.h | 16 +-------------- kernel/glo.h | 1 + kernel/proc.c | 36 +++++++++++++++++++++++++++++++++- kernel/proc.h | 1 - kernel/proto.h | 4 ++++ kernel/system/do_clear.c | 4 ++++ kernel/system/do_exec.c | 3 +++ kernel/system/do_fork.c | 5 +++++ kernel/system/do_mcontext.c | 8 ++++++++ kernel/system/do_sigreturn.c | 3 +++ kernel/system/do_sigsend.c | 8 +++++++- 14 files changed, 87 insertions(+), 49 deletions(-) diff --git a/kernel/arch/i386/arch_system.c b/kernel/arch/i386/arch_system.c index d336c6e70..6894c2440 100644 --- a/kernel/arch/i386/arch_system.c +++ b/kernel/arch/i386/arch_system.c @@ -206,42 +206,17 @@ PUBLIC void save_fpu(struct proc *pr) if(!fpu_presence) return; - /* If the process hasn't touched the FPU, there is nothing to do. */ - - if(!(pr->p_misc_flags & MF_USED_FPU)) - return; - /* Save changed FPU context. */ - if(osfxsr_feature) { fxsave(pr->p_fpu_state.fpu_save_area_p); fninit(); } else { fnsave(pr->p_fpu_state.fpu_save_area_p); } - - /* Clear MF_USED_FPU to signal there is no unsaved FPU state. */ - - pr->p_misc_flags &= ~MF_USED_FPU; } PUBLIC void restore_fpu(struct proc *pr) { - /* If the process hasn't touched the FPU, enable the FPU exception - * and don't restore anything. - */ - if(!(pr->p_misc_flags & MF_USED_FPU)) { - write_cr0(read_cr0() | I386_CR0_TS); - return; - } - - /* If the process has touched the FPU, disable the FPU - * exception (both for the kernel and for the process once - * it's scheduled), and initialize or restore the FPU state. - */ - - clts(); - if(!(pr->p_misc_flags & MF_FPU_INITIALIZED)) { fninit(); pr->p_misc_flags |= MF_FPU_INITIALIZED; @@ -496,6 +471,7 @@ PUBLIC struct proc * arch_finish_switch_to_user(void) stk = (char *)tss.sp0; /* set pointer to the process to run on the stack */ *((reg_t *)stk) = (reg_t) proc_ptr; + return proc_ptr; } diff --git a/kernel/arch/i386/exception.c b/kernel/arch/i386/exception.c index 89c5dbcb5..eed02883e 100644 --- a/kernel/arch/i386/exception.c +++ b/kernel/arch/i386/exception.c @@ -10,6 +10,7 @@ #include #include "kernel/proc.h" #include "kernel/proto.h" +#include extern int catch_pagefaults = 0; @@ -280,3 +281,14 @@ PUBLIC void proc_stacktrace(struct proc *whichproc) } printf("\n"); } + +PUBLIC void enable_fpu_exception(void) +{ + write_cr0(read_cr0() | I386_CR0_TS); +} + +PUBLIC void disable_fpu_exception(void) +{ + clts(); +} + diff --git a/kernel/arch/i386/mpx.S b/kernel/arch/i386/mpx.S index 487078fc9..a644d7669 100644 --- a/kernel/arch/i386/mpx.S +++ b/kernel/arch/i386/mpx.S @@ -65,6 +65,7 @@ begbss: */ .globl _restore_user_context +.globl _copr_not_available_handler .globl _reload_cr3 .globl _divide_error @@ -590,16 +591,12 @@ _inval_opcode: _copr_not_available: TEST_INT_IN_KERNEL(4, copr_not_available_in_kernel) cld /* set direction flag to a known value */ - SAVE_PROCESS_CTX_NON_LAZY(0) + SAVE_PROCESS_CTX(0) /* stop user process cycles */ push %ebp mov $0, %ebp call _context_stop - pop %ebp - lea P_MISC_FLAGS(%ebp), %ebx - orw $MF_USED_FPU, (%ebx) - mov $0, %ebp - jmp _switch_to_user + jmp _copr_not_available_handler copr_not_available_in_kernel: pushl $0 diff --git a/kernel/arch/i386/sconst.h b/kernel/arch/i386/sconst.h index 0b8cc910a..50e92f97b 100644 --- a/kernel/arch/i386/sconst.h +++ b/kernel/arch/i386/sconst.h @@ -121,7 +121,7 @@ * displ is the stack displacement. In case of an exception, there are two extra * value on the stack - error code and the exception number */ -#define SAVE_PROCESS_CTX_NON_LAZY(displ) \ +#define SAVE_PROCESS_CTX(displ) \ \ cld /* set the direction flag to a known state */ ;\ \ @@ -139,20 +139,6 @@ RESTORE_KERNEL_SEGS ;\ SAVE_TRAP_CTX(displ, %ebp, %esi) ; -#define SAVE_PROCESS_CTX(displ) \ - SAVE_PROCESS_CTX_NON_LAZY(displ) ;\ - push %eax ;\ - push %ebx ;\ - push %ecx ;\ - push %edx ;\ - push %ebp ;\ - call _save_fpu ;\ - pop %ebp ;\ - pop %edx ;\ - pop %ecx ;\ - pop %ebx ;\ - pop %eax ; - /* * clear the IF flag in eflags which are stored somewhere in memory, e.g. on * stack. iret or popf will load the new value later diff --git a/kernel/glo.h b/kernel/glo.h index e2e841411..ae8a58385 100644 --- a/kernel/glo.h +++ b/kernel/glo.h @@ -45,6 +45,7 @@ EXTERN time_t boottime; EXTERN char params_buffer[512]; /* boot monitor parameters */ EXTERN int minix_panicing; EXTERN char fpu_presence; +EXTERN struct proc * fpu_owner; EXTERN int verboseboot; /* verbose boot, init'ed in cstart */ #define MAGICTEST 0xC0FFEE23 EXTERN u32_t magictest; /* global magic number */ diff --git a/kernel/proc.c b/kernel/proc.c index 85fb811f7..dc4eda719 100644 --- a/kernel/proc.c +++ b/kernel/proc.c @@ -237,11 +237,15 @@ check_misc_flags: context_stop(proc_addr(KERNEL)); + /* If the process isn't the owner of FPU, enable the FPU exception */ + if(fpu_owner != proc_ptr) + enable_fpu_exception(); + else + disable_fpu_exception(); /* * restore_user_context() carries out the actual mode switch from kernel * to userspace. This function does not return */ - restore_fpu(proc_ptr); restore_user_context(proc_ptr); NOT_REACHABLE; } @@ -1433,3 +1437,33 @@ PUBLIC void proc_no_time(struct proc * p) #endif } } + +PUBLIC void copr_not_available_handler(void) +{ + /* + * Disable the FPU exception (both for the kernel and for the process + * once it's scheduled), and initialize or restore the FPU state. + */ + + disable_fpu_exception(); + + /* if FPU is not owned by anyone, do not store anything */ + if (fpu_owner != NULL) { + assert(fpu_owner != proc_ptr); + save_fpu(fpu_owner); + } + + /* + * restore the current process' state and let it run again, do not + * schedule! + */ + restore_fpu(proc_ptr); + fpu_owner = proc_ptr; + context_stop(proc_addr(KERNEL)); + restore_user_context(proc_ptr); + NOT_REACHABLE; +} + +PUBLIC void release_fpu(void) { + fpu_owner = NULL; +} diff --git a/kernel/proc.h b/kernel/proc.h index f8a465647..7a239edcc 100644 --- a/kernel/proc.h +++ b/kernel/proc.h @@ -221,7 +221,6 @@ struct proc { #define MF_SC_ACTIVE 0x100 /* Syscall tracing: in a system call now */ #define MF_SC_DEFER 0x200 /* Syscall tracing: deferred system call */ #define MF_SC_TRACE 0x400 /* Syscall tracing: trigger syscall events */ -#define MF_USED_FPU 0x800 /* process used fpu during last execution run */ #define MF_FPU_INITIALIZED 0x1000 /* process already used math, so fpu * regs are significant (initialized)*/ #define MF_SENDING_FROM_KERNEL 0x2000 /* message of this process is from kernel */ diff --git a/kernel/proto.h b/kernel/proto.h index 32ea89448..ac6e95a17 100644 --- a/kernel/proto.h +++ b/kernel/proto.h @@ -189,6 +189,10 @@ _PROTOTYPE( int copy_msg_to_user, (struct proc * p, message * src, _PROTOTYPE(void switch_address_space, (struct proc * p)); _PROTOTYPE(void release_address_space, (struct proc *pr)); +_PROTOTYPE(void enable_fpu_exception, (void)); +_PROTOTYPE(void disable_fpu_exception, (void)); +_PROTOTYPE(void release_fpu, (void)); + /* utility.c */ _PROTOTYPE( void cpu_print_freq, (unsigned cpu)); #endif /* PROTO_H */ diff --git a/kernel/system/do_clear.c b/kernel/system/do_clear.c index db960cb7a..056bbf243 100644 --- a/kernel/system/do_clear.c +++ b/kernel/system/do_clear.c @@ -70,6 +70,10 @@ PUBLIC int do_clear(struct proc * caller, message * m_ptr) } #endif + /* release FPU */ + if (fpu_owner == rc) + release_fpu(); + return OK; } diff --git a/kernel/system/do_exec.c b/kernel/system/do_exec.c index 0e9e473c9..e6fe9892f 100644 --- a/kernel/system/do_exec.c +++ b/kernel/system/do_exec.c @@ -46,6 +46,9 @@ PUBLIC int do_exec(struct proc * caller, message * m_ptr) /* Mark fpu_regs contents as not significant, so fpu * will be initialized, when it's used next time. */ rp->p_misc_flags &= ~MF_FPU_INITIALIZED; + /* force reloading FPU if the current process is the owner */ + if (rp == fpu_owner) + release_fpu(); return(OK); } #endif /* USE_EXEC */ diff --git a/kernel/system/do_fork.c b/kernel/system/do_fork.c index 11629afb9..302e7262b 100644 --- a/kernel/system/do_fork.c +++ b/kernel/system/do_fork.c @@ -51,6 +51,11 @@ PUBLIC int do_fork(struct proc * caller, message * m_ptr) map_ptr= (struct mem_map *) m_ptr->PR_MEM_PTR; + /* make sure that the FPU context is saved in parent before copy */ + if (fpu_owner == rpp) { + disable_fpu_exception(); + save_fpu(rpp); + } /* Copy parent 'proc' struct to child. And reinitialize some fields. */ gen = _ENDPOINT_G(rpc->p_endpoint); #if (_MINIX_CHIP == _CHIP_INTEL) diff --git a/kernel/system/do_mcontext.c b/kernel/system/do_mcontext.c index 0dd86b83d..fa0ed772d 100644 --- a/kernel/system/do_mcontext.c +++ b/kernel/system/do_mcontext.c @@ -42,6 +42,11 @@ PUBLIC int do_getmcontext(struct proc * caller, message * m_ptr) /* Copy FPU state */ mc.mc_fpu_flags = 0; if (rp->p_misc_flags & MF_FPU_INITIALIZED) { + /* make sure that the FPU context is saved into proc structure first */ + if (fpu_owner == rp) { + disable_fpu_exception(); + save_fpu(rp); + } mc.mc_fpu_flags = 0 | rp->p_misc_flags & MF_FPU_INITIALIZED; memcpy(&(mc.mc_fpu_state), rp->p_fpu_state.fpu_save_area_p, FPU_XFP_SIZE); @@ -86,6 +91,9 @@ PUBLIC int do_setmcontext(struct proc * caller, message * m_ptr) FPU_XFP_SIZE); } else rp->p_misc_flags &= ~MF_FPU_INITIALIZED; + /* force reloading FPU in either case */ + if (fpu_owner == rp) + release_fpu(); #endif return(OK); diff --git a/kernel/system/do_sigreturn.c b/kernel/system/do_sigreturn.c index 5224de2a6..6e8e1abe1 100644 --- a/kernel/system/do_sigreturn.c +++ b/kernel/system/do_sigreturn.c @@ -59,6 +59,9 @@ PUBLIC int do_sigreturn(struct proc * caller, message * m_ptr) memcpy(rp->p_fpu_state.fpu_save_area_p, &sc.sc_fpu_state, FPU_XFP_SIZE); rp->p_misc_flags |= MF_FPU_INITIALIZED; /* Restore math usage flag. */ + /* force reloading FPU */ + if (fpu_owner == rp) + release_fpu(); } #endif diff --git a/kernel/system/do_sigsend.c b/kernel/system/do_sigsend.c index 656a86fb6..6da9463a0 100644 --- a/kernel/system/do_sigsend.c +++ b/kernel/system/do_sigsend.c @@ -44,9 +44,15 @@ PUBLIC int do_sigsend(struct proc * caller, message * m_ptr) /* Copy the registers to the sigcontext structure. */ memcpy(&sc.sc_regs, (char *) &rp->p_reg, sizeof(sigregs)); #if (_MINIX_CHIP == _CHIP_INTEL) - if(rp->p_misc_flags & MF_FPU_INITIALIZED) + if(rp->p_misc_flags & MF_FPU_INITIALIZED) { + /* save the FPU context before saving it to the sig context */ + if (fpu_owner == rp) { + disable_fpu_exception(); + save_fpu(rp); + } memcpy(&sc.sc_fpu_state, rp->p_fpu_state.fpu_save_area_p, FPU_XFP_SIZE); + } #endif /* Finish the sigcontext initialization. */ -- 2.44.0