Commit bc1790d7 authored by Steven Seeger's avatar Steven Seeger
Browse files

enable ipipe

parent fe76bc84
......@@ -11,6 +11,7 @@ config 32BIT
config 64BIT
bool
depends on !IPIPE
default y if PPC64
config ARCH_PHYS_ADDR_T_64BIT
......@@ -223,6 +224,7 @@ config PPC
select HAVE_IPIPE_SUPPORT
select IRQ_DOMAIN
select IRQ_FORCED_THREADING
select IPIPE_HAVE_HOSTRT if IPIPE
select MODULES_USE_ELF_RELA
select NO_BOOTMEM
select OF
......@@ -394,6 +396,18 @@ source "arch/powerpc/platforms/Kconfig"
menu "Kernel options"
config IPIPE_HAVE_PREEMPTIBLE_SWITCH
bool
depends on IPIPE
default y
source "kernel/ipipe/Kconfig"
config IPIPE_HAVE_PREEMPTIBLE_SWITCH
bool
depends on IPIPE
default y
config HIGHMEM
bool "High memory support"
depends on PPC32
......
......@@ -58,6 +58,14 @@ ifdef CONFIG_DEBUG_INFO
BOOTCFLAGS += -g
endif
ifdef CONFIG_IPIPE_TRACE
# do not trace the boot loader
nullstring :=
space := $(nullstring) # end of the line
pg_flag = $(nullstring) -pg # end of the line
BOOTCFLAGS := $(subst ${pg_flag},${space},${BOOTCFLAGS})
endif
ifeq ($(call cc-option-yn, -fstack-protector),y)
BOOTCFLAGS += -fno-stack-protector
endif
......
......@@ -141,6 +141,8 @@ extern void force_external_irq_replay(void);
#define SET_MSR_EE(x) mtmsr(x)
#ifndef CONFIG_IPIPE
static inline unsigned long arch_local_save_flags(void)
{
return mfmsr();
......@@ -201,7 +203,9 @@ static inline bool arch_irqs_disabled(void)
return arch_irqs_disabled_flags(arch_local_save_flags());
}
#define hard_irq_disable() arch_local_irq_disable()
#endif /* !CONFIG_IPIPE */
#define hard_irq_disable() hard_local_irq_disable()
static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
{
......@@ -214,6 +218,7 @@ static inline void may_hard_irq_enable(void) { }
#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST
#include <asm/ipipe_hwirq.h>
/*
* interrupt-retrigger: should we handle this via lost interrupts and IPIs
* or should we not care like we do now ? --BenH.
......
......@@ -67,8 +67,15 @@ extern void call_do_softirq(struct thread_info *tp);
extern void call_do_irq(struct pt_regs *regs, struct thread_info *tp);
extern void do_IRQ(struct pt_regs *regs);
extern void __do_irq(struct pt_regs *regs);
extern void ___do_irq(unsigned int irq, struct pt_regs *regs);
int irq_choose_cpu(const struct cpumask *mask);
#if defined(CONFIG_DEBUG_STACKOVERFLOW)
void check_stack_overflow(void);
#else
static inline void check_stack_overflow(void) {}
#endif
#endif /* _ASM_IRQ_H */
#endif /* __KERNEL__ */
......@@ -39,35 +39,9 @@
#define TRACE_ENABLE_INTS TRACE_WITH_FRAME_BUFFER(trace_hardirqs_on)
#define TRACE_DISABLE_INTS TRACE_WITH_FRAME_BUFFER(trace_hardirqs_off)
/*
* This is used by assembly code to soft-disable interrupts first and
* reconcile irq state.
*
* NB: This may call C code, so the caller must be prepared for volatiles to
* be clobbered.
*/
#define RECONCILE_IRQ_STATE(__rA, __rB) \
lbz __rA,PACASOFTIRQEN(r13); \
lbz __rB,PACAIRQHAPPENED(r13); \
cmpwi cr0,__rA,0; \
li __rA,0; \
ori __rB,__rB,PACA_IRQ_HARD_DIS; \
stb __rB,PACAIRQHAPPENED(r13); \
beq 44f; \
stb __rA,PACASOFTIRQEN(r13); \
TRACE_DISABLE_INTS; \
44:
#else
#define TRACE_ENABLE_INTS
#define TRACE_DISABLE_INTS
#define RECONCILE_IRQ_STATE(__rA, __rB) \
lbz __rA,PACAIRQHAPPENED(r13); \
li __rB,0; \
ori __rA,__rA,PACA_IRQ_HARD_DIS; \
stb __rB,PACASOFTIRQEN(r13); \
stb __rA,PACAIRQHAPPENED(r13)
#endif
#endif
......
......@@ -49,7 +49,7 @@ static inline void klp_init_thread_info(struct thread_info *ti)
ti->livepatch_sp = (unsigned long *)(ti + 1) + 1;
}
#else
static void klp_init_thread_info(struct thread_info *ti) { }
static inline void klp_init_thread_info(struct thread_info *ti) { }
#endif /* CONFIG_LIVEPATCH */
#endif /* _ASM_POWERPC_LIVEPATCH_H */
......@@ -7,6 +7,8 @@
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
#include <linux/ipipe.h>
#include <linux/ipipe_debug.h>
#include <asm/mmu.h>
#include <asm/cputable.h>
#include <asm/cputhreads.h>
......@@ -78,21 +80,84 @@ extern void switch_cop(struct mm_struct *next);
extern int use_cop(unsigned long acop, struct mm_struct *mm);
extern void drop_cop(unsigned long acop, struct mm_struct *mm);
extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk);
extern void __do_switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk, bool irq_sync_p);
static inline void __switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH
#if defined(CONFIG_PPC_MMU_NOHASH) && defined(CONFIG_SMP)
/*
* mmu_context_nohash in SMP mode is tracking an activity counter
* into the mm struct. Therefore, we make sure the kernel always sees
* the ipipe_percpu.active_mm update and the actual switch as a
* single atomic operation. Since the related code already requires
* to hard disable irqs all through the switch, there is no
* additional penalty anyway.
*/
#define mmswitch_irq_sync false
#else
#define mmswitch_irq_sync true
#endif
IPIPE_WARN_ONCE(hard_irqs_disabled());
for (;;) {
hard_local_irq_disable();
__this_cpu_write(ipipe_percpu.active_mm, NULL);
barrier();
__do_switch_mm(prev, next, tsk, mmswitch_irq_sync);
if (!test_and_clear_thread_flag(TIF_MMSWITCH_INT)) {
__this_cpu_write(ipipe_percpu.active_mm, next);
hard_local_irq_enable();
return;
}
}
#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
IPIPE_WARN_ONCE(!hard_irqs_disabled());
__do_switch_mm(prev, next, tsk, false);
#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */
}
static inline void switch_mm_irqs_off(struct mm_struct *prev,
struct mm_struct *next,
struct task_struct *tsk)
{
unsigned long flags;
ipipe_mm_switch_protect(flags);
__switch_mm(prev, next, tsk);
ipipe_mm_switch_unprotect(flags);
}
#ifdef CONFIG_IPIPE
/*
* ipipe_switch_mm_head() is reserved to the head domain for switching
* mmu context.
*/
static inline
void ipipe_switch_mm_head(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
__do_switch_mm(prev, next, tsk, false);
}
#endif /* CONFIG_IPIPE */
static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
#ifndef CONFIG_IPIPE
unsigned long flags;
local_irq_save(flags);
#endif
switch_mm_irqs_off(prev, next, tsk);
#ifndef CONFIG_IPIPE
local_irq_restore(flags);
#endif
}
#define switch_mm_irqs_off switch_mm_irqs_off
#define deactivate_mm(tsk,mm) do { } while (0)
/*
......
......@@ -293,7 +293,7 @@ struct mpic
#ifdef CONFIG_MPIC_U3_HT_IRQS
/* The fixup table */
struct mpic_irq_fixup *fixups;
raw_spinlock_t fixup_lock;
ipipe_spinlock_t fixup_lock;
#endif
/* Register access method */
......
......@@ -50,6 +50,7 @@ extern unsigned int debug_smp_processor_id(void); /* from linux/smp.h */
#define get_slb_shadow() (get_paca()->slb_shadow_ptr)
struct task_struct;
struct ipipe_percpu_domain_data;
/*
* Defines the layout of the paca.
......@@ -159,8 +160,12 @@ struct paca_struct {
u64 saved_r1; /* r1 save for RTAS calls or PM */
u64 saved_msr; /* MSR saved here by enter_rtas */
u16 trap_save; /* Used when bad stack is encountered */
#ifdef CONFIG_IPIPE
u64 ipipe_statp; /* Pointer to I-pipe status word */
#else
u8 soft_enabled; /* irq soft-enable flag */
u8 irq_happened; /* irq happened while soft-disabled */
#endif
u8 io_sync; /* writel() needs spin_unlock sync */
u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */
u8 nap_state_lost; /* NV GPR values lost in power7_idle */
......@@ -212,6 +217,10 @@ struct paca_struct {
u8 hmi_event_available; /* HMI event is available */
#endif
#ifdef CONFIG_IPIPE
struct ipipe_percpu_domain_data *root_context; /* Address of root context data */
#endif /* CONFIG_IPIPE */
/* Stuff for accurate time accounting */
struct cpu_accounting_data accounting;
u64 dtl_ridx; /* read index in dispatch log */
......
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_POWERPC_PERCPU_H_
#define _ASM_POWERPC_PERCPU_H_
#include <asm/ipipe_base.h>
#ifdef __powerpc64__
/*
......
......@@ -35,6 +35,7 @@
#define MSR_TS_LG 33 /* Trans Mem state (2 bits) */
#define MSR_TM_LG 32 /* Trans Mem Available */
#define MSR_VEC_LG 25 /* Enable AltiVec */
#define MSR_VIRTEE_LG 29 /* I-pipe stall bit */
#define MSR_VSX_LG 23 /* Enable VSX */
#define MSR_POW_LG 18 /* Enable Power Management */
#define MSR_WE_LG 18 /* Wait State Enable */
......@@ -121,6 +122,11 @@
#define MSR_TM_TRANSACTIONAL(x) (((x) & MSR_TS_MASK) == MSR_TS_T)
#define MSR_TM_SUSPENDED(x) (((x) & MSR_TS_MASK) == MSR_TS_S)
/*
* CONFIG_IPIPE only. We divert the unused bit #29 from the MSR.
*/
#define MSR_VIRTEE __MASK(MSR_VIRTEE_LG)
#if defined(CONFIG_PPC_BOOK3S_64)
#define MSR_64BIT MSR_SF
......
......@@ -125,6 +125,7 @@ extern int cpu_to_core_id(int cpu);
#define PPC_MSG_RESCHEDULE 1
#define PPC_MSG_TICK_BROADCAST 2
#define PPC_MSG_NMI_IPI 3
#define PPC_MSG_IPIPE_DEMUX PPC_MSG_NMI_IPI
/* This is only used by the powernv kernel */
#define PPC_MSG_RM_HOST_ACTION 4
......
......@@ -27,6 +27,7 @@
#include <asm/page.h>
#include <linux/stringify.h>
#include <asm/accounting.h>
#include <ipipe/thread_info.h>
/*
* low level task data.
......@@ -45,6 +46,10 @@ struct thread_info {
#endif
/* low level flags - has atomic operations done on it */
unsigned long flags ____cacheline_aligned_in_smp;
#ifdef CONFIG_IPIPE
unsigned long ipipe_flags;
#endif
struct ipipe_threadinfo ipipe_data;
};
/*
......@@ -101,6 +106,7 @@ static inline struct thread_info *current_thread_info(void)
#if defined(CONFIG_PPC64)
#define TIF_ELF2ABI 18 /* function descriptors must die! */
#endif
#define TIF_MMSWITCH_INT 20 /* MMU context switch interrupted */
/* as above, but as bit values */
#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
......@@ -129,6 +135,15 @@ static inline struct thread_info *current_thread_info(void)
_TIF_RESTORE_TM | _TIF_PATCH_PENDING)
#define _TIF_PERSYSCALL_MASK (_TIF_RESTOREALL|_TIF_NOERROR)
/* ti->ipipe_flags */
#define TIP_MAYDAY 0 /* MAYDAY call is pending */
#define TIP_NOTIFY 1 /* Notify head domain about kernel events */
#define TIP_HEAD 2 /* Runs in head domain */
#define _TIP_MAYDAY (1<<TIP_MAYDAY)
#define _TIP_NOTIFY (1<<TIP_NOTIFY)
#define _TIP_HEAD (1<<TIP_HEAD)
/* Bits in local_flags */
/* Don't move TLF_NAPPING without adjusting the code in entry_32.S */
#define TLF_NAPPING 0 /* idle thread enabled NAP mode */
......
......@@ -7,6 +7,7 @@
#include <asm/processor.h>
#include <asm/page.h>
#include <asm/extable.h>
#include <asm-generic/ipipe.h>
/*
* The fs value determines whether argument validity checking should be
......@@ -91,6 +92,21 @@
#define __put_user_inatomic(x, ptr) \
__put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
#ifdef CONFIG_IPIPE
/*
* Calling from atomic context should be allowed for all __nocheck
* accessors. The head domain may do this, provided page faulting is
* duly prevented.
*/
#define __chk_might_fault(__pu) do { } while (0)
#else
#define __chk_might_fault(__pu) \
do { \
if (!is_kernel_addr((unsigned long)__pu)) \
might_fault(); \
} while (0)
#endif
extern long __put_user_bad(void);
/*
......@@ -145,8 +161,7 @@ do { \
({ \
long __pu_err; \
__typeof__(*(ptr)) __user *__pu_addr = (ptr); \
if (!is_kernel_addr((unsigned long)__pu_addr)) \
might_fault(); \
__chk_might_fault(__pu_addr); \
__chk_user_ptr(ptr); \
__put_user_size((x), __pu_addr, (size), __pu_err); \
__pu_err; \
......
......@@ -51,6 +51,7 @@ obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_book3e.o
obj-$(CONFIG_PPC_BARRIER_NOSPEC) += security.o
obj-$(CONFIG_PPC64) += vdso64/
obj-$(CONFIG_ALTIVEC) += vecemu.o
obj-$(CONFIG_IPIPE) += ipipe.o
obj-$(CONFIG_PPC_970_NAP) += idle_power4.o
obj-$(CONFIG_PPC_P7_NAP) += idle_book3s.o
procfs-y := proc_powerpc.o
......
......@@ -155,6 +155,9 @@ int main(void)
OFFSET(TI_FLAGS, thread_info, flags);
OFFSET(TI_LOCAL_FLAGS, thread_info, local_flags);
#ifdef CONFIG_IPIPE
OFFSET(TI_IPIPE, thread_info, ipipe_flags);
#endif
OFFSET(TI_PREEMPT, thread_info, preempt_count);
OFFSET(TI_TASK, thread_info, task);
OFFSET(TI_CPU, thread_info, cpu);
......@@ -178,6 +181,9 @@ int main(void)
OFFSET(PACATOC, paca_struct, kernel_toc);
OFFSET(PACAKBASE, paca_struct, kernelbase);
OFFSET(PACAKMSR, paca_struct, kernel_msr);
#ifdef CONFIG_IPIPE
OFFSET(PACAROOTPCPU, paca_struct, ipipe_statp);
#endif
OFFSET(PACASOFTIRQEN, paca_struct, soft_enabled);
OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
#ifdef CONFIG_PPC_BOOK3S
......
......@@ -2262,7 +2262,7 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
return t;
}
struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr)
notrace struct cpu_spec * __init identify_cpu(unsigned long offset, unsigned int pvr)
{
struct cpu_spec *s = cpu_specs;
int i;
......
......@@ -324,6 +324,66 @@ _GLOBAL(DoSyscall)
lwz r11,_CCR(r1) /* Clear SO bit in CR */
rlwinm r11,r11,0,4,2
stw r11,_CCR(r1)
#ifdef CONFIG_IPIPE
CURRENT_THREAD_INFO(r10, r1)
cmplwi cr0,r0,NR_syscalls
blt slow_path
lwz r11,TI_IPIPE(r10)
andi. r12,r11,_TIP_HEAD /* implies that TIP_NOTIFY if set */
beq slow_path_check
addi r3,r1,GPR0
bl ipipe_fastcall_hook /* __IPIPE_SYSCALL_E is assumed */
cmpwi r3,0
blt no_fastcall
CURRENT_THREAD_INFO(r10, r1)
lwz r11,TI_IPIPE(r10)
andi. r12,r11,_TIP_HEAD
bne+ fastcall_exit_check
bl __ipipe_root_sync
lwz r3,GPR3(r1)
b ret_from_syscall
no_fastcall:
/* fastcall handler not implemented */
lwz r0,GPR0(r1)
slow_path:
lwz r11,TI_IPIPE(r10)
slow_path_check:
andi. r12,r11,_TIP_NOTIFY
bne pipeline_syscall
cmplwi cr0,r0,NR_syscalls
blt root_syscall
pipeline_syscall:
addi r3,r1,GPR0
bl __ipipe_notify_syscall
cmpwi cr1,r3,0
CURRENT_THREAD_INFO(r10, r1)
lwz r11,TI_IPIPE(r10)
andi. r12,r11,_TIP_HEAD
bne fastcall_exit
beq cr1,root_syscall
lwz r3,GPR3(r1) /* syscall return value */
b ret_from_syscall
fastcall_exit_check:
andi. r12,r11,_TIP_MAYDAY
beq+ fastcall_exit
addi r3,r1,GPR0
bl __ipipe_call_mayday
fastcall_exit:
lwz r3,GPR3(r1) /* syscall return value */
LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* clear MSR_EE */
SYNC
MTMSRD(r10)
b syscall_exit_cont
root_syscall:
lwz r0,GPR0(r1)
lwz r3,GPR3(r1)
lwz r4,GPR4(r1)
lwz r5,GPR5(r1)
lwz r6,GPR6(r1)
lwz r7,GPR7(r1)
lwz r8,GPR8(r1)
lwz r9,GPR9(r1)
#endif /* CONFIG_IPIPE */
#ifdef CONFIG_TRACE_IRQFLAGS
/* Return from syscalls can (and generally will) hard enable
* interrupts. You aren't supposed to call a syscall with
......@@ -459,6 +519,21 @@ END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
.globl ret_from_fork
ret_from_fork:
#ifdef CONFIG_IPIPE
#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
stwu r1,-4(r1)
stw r3,0(r1)
lis r3,(0x80000000)@h
ori r3,r3,(0x80000000)@l
bl ipipe_trace_end
lwz r3,0(r1)
addi r1,r1,4
#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
LOAD_MSR_KERNEL(r10,MSR_KERNEL)
ori r10,r10,MSR_EE
SYNC
MTMSRD(r10)
#endif /* CONFIG_IPIPE */
REST_NVGPRS(r1)
bl schedule_tail
li r3,0
......@@ -466,6 +541,21 @@ ret_from_fork:
.globl ret_from_kernel_thread
ret_from_kernel_thread:
#ifdef CONFIG_IPIPE
#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
stwu r1,-4(r1)
stw r3,0(r1)
lis r3,(0x80000000)@h
ori r3,r3,(0x80000000)@l
bl ipipe_trace_end
lwz r3,0(r1)
addi r1,r1,4
#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
LOAD_MSR_KERNEL(r10,MSR_KERNEL)
ori r10,r10,MSR_EE
SYNC
MTMSRD(r10)
#endif /* CONFIG_IPIPE */
REST_NVGPRS(r1)
bl schedule_tail
mtlr r14
......@@ -807,6 +897,13 @@ ret_from_except:
SYNC /* Some chip revs have problems here... */
MTMSRD(r10) /* disable interrupts */
#ifdef CONFIG_IPIPE
CURRENT_THREAD_INFO(r9, r1)
lwz r0,TI_IPIPE(r9)
andi. r0,r0,_TIP_HEAD
bne- restore
#endif /* CONFIG_IPIPE */
lwz r3,_MSR(r1) /* Returning to user mode? */
andi. r0,r3,MSR_PR
beq resume_kernel
......@@ -833,6 +930,12 @@ restore_user:
b restore
#ifdef CONFIG_IPIPE
#define PREEMPT_SCHEDULE_IRQ __ipipe_preempt_schedule_irq
#else
#define PREEMPT_SCHEDULE_IRQ preempt_schedule_irq
#endif
/* N.B. the only way to get here is from the beq following ret_from_except. */
resume_kernel:
/* check current_thread_info, _TIF_EMULATE_STACK_STORE */
......@@ -890,7 +993,7 @@ resume_kernel:
*/
bl trace_hardirqs_off
#endif
1: bl preempt_schedule_irq
1: bl PREEMPT_SCHEDULE_IRQ
CURRENT_THREAD_INFO(r9, r1)
lwz r3,TI_FLAGS(r9)
andi. r0,r3,_TIF_NEED_RESCHED
......@@ -1307,6 +1410,15 @@ ee_restarts:
.space 4
.previous
#ifdef CONFIG_IPIPE
_GLOBAL(__ipipe_ret_from_except_full)
REST_NVGPRS(r1)
_GLOBAL(__ipipe_ret_from_except)
cmpwi r3,0
bne+ ret_from_except
b restore
#endif /* CONFIG_IPIPE */
/*
* PROM code for specific machines follows. Put it
* here so it's easy to add arch-specific sections later.
......
......@@ -45,6 +45,10 @@
#include <asm/exception-64e.h>
#endif
#ifdef CONFIG_IPIPE
#error PowerPC-64 support was removed from I-Pipe.
#endif
/*
* System calls.
*/
......
......@@ -326,6 +326,12 @@ i##n: \
EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
#ifdef CONFIG_IPIPE
#define EXC_XFER_IPIPE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
__ipipe_ret_from_except)
#endif /* CONFIG_IPIPE */
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
ret_from_except)
......@@ -413,7 +419,11 @@ InstructionAccess:
EXC_XFER_LITE(0x400, handle_page_fault)
/* External interrupt */
#ifdef CONFIG_IPIPE
EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
#else /* !CONFIG_IPIPE */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)