ARM: ipipe: add pipeline core

arch/arm/Kconfig

@@ -75,6 +75,7 @@ config ARM
 	select HAVE_HW_BREAKPOINT if (PERF_EVENTS && (CPU_V6 || CPU_V6K || CPU_V7))
 	select HAVE_IDE if PCI || ISA || PCMCIA
 	select HAVE_IRQ_TIME_ACCOUNTING
+	select HAVE_IPIPE_SUPPORT
 	select HAVE_KERNEL_GZIP
 	select HAVE_KERNEL_LZ4
 	select HAVE_KERNEL_LZMA
@@ -1482,6 +1483,8 @@ config ARCH_NR_GPIO
 
 	  If unsure, leave the default value.
 
+source "kernel/ipipe/Kconfig"
+
 config HZ_FIXED
 	int
 	default 200 if ARCH_EBSA110
@@ -1750,6 +1753,7 @@ config ALIGNMENT_TRAP
 config UACCESS_WITH_MEMCPY
 	bool "Use kernel mem{cpy,set}() for {copy_to,clear}_user()"
 	depends on MMU
+	depends on !IPIPE
 	default y if CPU_FEROCEON
 	help
 	  Implement faster copy_to_user and clear_user methods for CPU

arch/arm/include/asm/assembler.h

@@ -100,6 +100,18 @@
 	.macro	enable_irq_notrace
 	cpsie	i
 	.endm
+
+	.macro  disable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsid	i
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro  enable_irq_cond
+#ifdef CONFIG_IPIPE
+	cpsie	i
+#endif /* CONFIG_IPIPE */
+	.endm
 #else
 	.macro	disable_irq_notrace
 	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
@@ -108,10 +120,22 @@
 	.macro	enable_irq_notrace
 	msr	cpsr_c, #SVC_MODE
 	.endm
+
+	.macro	disable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #PSR_I_BIT | SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
+
+	.macro	enable_irq_cond
+#ifdef CONFIG_IPIPE
+	msr	cpsr_c, #SVC_MODE
+#endif /* CONFIG_IPIPE */
+	.endm
 #endif
 
 	.macro asm_trace_hardirqs_off, save=1
-#if defined(CONFIG_TRACE_IRQFLAGS)
+#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_IPIPE)
 	.if \save
 	stmdb   sp!, {r0-r3, ip, lr}
 	.endif
@@ -123,7 +147,7 @@
 	.endm
 
 	.macro asm_trace_hardirqs_on, cond=al, save=1
-#if defined(CONFIG_TRACE_IRQFLAGS)
+#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_IPIPE)
 	/*
 	 * actually the registers should be pushed and pop'd conditionally, but
 	 * after bl the flags are certainly clobbered

arch/arm/include/asm/entry-macro-multi.S

@@ -12,7 +12,11 @@
 	@ routine called with r0 = irq number, r1 = struct pt_regs *
 	@
 	badrne	lr, 1b
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_irq
+#else
 	bne	asm_do_IRQ
+#endif
 
 #ifdef CONFIG_SMP
 	/*
@@ -25,8 +29,12 @@
 	ALT_UP_B(9997f)
 	movne	r1, sp
 	badrne	lr, 1b
+#ifdef CONFIG_IPIPE
+	bne	__ipipe_grab_ipi
+#else
 	bne	do_IPI
 #endif
+#endif
 9997:
 	.endm
 

arch/arm/include/asm/hw_irq.h

@@ -14,4 +14,8 @@ static inline void ack_bad_irq(int irq)
 
 #define ARCH_IRQ_INIT_FLAGS	(IRQ_NOREQUEST | IRQ_NOPROBE)
 
+#define IPIPE_NR_ROOT_IRQS	1024
+
+#define IPIPE_NR_XIRQS		IPIPE_NR_ROOT_IRQS
+
 #endif

arch/arm/include/asm/ipipe.h

+/* -*- linux-c -*-
+ * arch/arm/include/asm/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ARM_IPIPE_H
+#define __ARM_IPIPE_H
+
+#include <linux/irqdomain.h>
+
+#ifdef CONFIG_IPIPE
+
+#define BROKEN_BUILTIN_RETURN_ADDRESS
+#undef __BUILTIN_RETURN_ADDRESS0
+#undef __BUILTIN_RETURN_ADDRESS1
+#ifdef CONFIG_FRAME_POINTER
+#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0)
+#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1)
+extern unsigned long arm_return_addr(int level);
+#else
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 (0)
+#endif
+
+#include <linux/jump_label.h>
+#include <linux/ipipe_trace.h>
+
+#define IPIPE_CORE_RELEASE	1
+
+struct ipipe_domain;
+struct timekeeper;
+
+#define IPIPE_TSC_TYPE_NONE	   		0
+#define IPIPE_TSC_TYPE_FREERUNNING 		1
+#define IPIPE_TSC_TYPE_DECREMENTER 		2
+#define IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN	3
+#define IPIPE_TSC_TYPE_FREERUNNING_TWICE	4
+#define IPIPE_TSC_TYPE_FREERUNNING_ARCH		5
+
+/* tscinfo, exported to user-space */
+struct __ipipe_tscinfo {
+	unsigned type;
+	unsigned freq;
+	unsigned long counter_vaddr;
+	union {
+		struct {
+			unsigned long counter_paddr;
+			unsigned long long mask;
+		};
+		struct {
+			unsigned *counter; /* Hw counter physical address */
+			unsigned long long mask; /* Significant bits in the hw counter. */
+			unsigned long long *tsc; /* 64 bits tsc value. */
+		} fr;
+		struct {
+			unsigned *counter; /* Hw counter physical address */
+			unsigned long long mask; /* Significant bits in the hw counter. */
+			unsigned *last_cnt; /* Counter value when updating
+						tsc value. */
+			unsigned long long *tsc; /* 64 bits tsc value. */
+		} dec;
+	} u;
+	unsigned int (*refresh_freq)(void);
+};
+
+struct ipipe_arch_sysinfo {
+	struct __ipipe_tscinfo tsc;
+};
+
+
+/* arch specific stuff */
+extern char __ipipe_tsc_area[];
+void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info);
+
+#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
+unsigned long long __ipipe_tsc_get(void) __attribute__((long_call));
+void __ipipe_tsc_register(struct __ipipe_tscinfo *info);
+void __ipipe_tsc_update(void);
+void __ipipe_update_vsyscall(struct timekeeper *tk);
+extern unsigned long __ipipe_kuser_tsc_freq;
+#define __ipipe_hrclock_freq __ipipe_kuser_tsc_freq
+#else /* ! generic tsc */
+unsigned long long __ipipe_mach_get_tsc(void);
+#define __ipipe_tsc_get() __ipipe_mach_get_tsc()
+static inline void __ipipe_update_vsyscall(struct timekeeper *tk) {}
+#ifndef __ipipe_hrclock_freq
+extern unsigned long __ipipe_hrtimer_freq;
+#define __ipipe_hrclock_freq __ipipe_hrtimer_freq
+#endif /* !__ipipe_mach_hrclock_freq */
+#endif /* ! generic tsc */
+
+#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
+extern void (*__ipipe_mach_hrtimer_debug)(unsigned irq);
+#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */
+
+#define ipipe_read_tsc(t)	do { t = __ipipe_tsc_get(); } while(0)
+#define __ipipe_read_timebase()	__ipipe_tsc_get()
+
+#define ipipe_tsc2ns(t) \
+({ \
+	unsigned long long delta = (t)*1000; \
+	do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+#define ipipe_tsc2us(t) \
+({ \
+	unsigned long long delta = (t); \
+	do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \
+	(unsigned long)delta; \
+})
+
+static inline const char *ipipe_clock_name(void)
+{
+	return "ipipe_tsc";
+}
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_enable_irq(irq)		enable_irq(irq)
+#define __ipipe_disable_irq(irq)	disable_irq(irq)
+
+static inline void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{ }
+
+static inline void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
+{ }
+
+#define ipipe_notify_root_preemption() do { } while(0)
+
+#ifdef CONFIG_SMP
+void __ipipe_early_core_setup(void);
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
+void __ipipe_root_localtimer(unsigned int irq, void *cookie);
+void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg);
+void __ipipe_do_vnmi(unsigned int irq, void *cookie);
+void __ipipe_grab_ipi(unsigned svc, struct pt_regs *regs);
+void __ipipe_ipis_alloc(void);
+void __ipipe_ipis_request(void);
+
+static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs)
+{
+	__ipipe_grab_ipi(irq, regs);
+}
+
+#ifdef CONFIG_SMP_ON_UP
+extern struct static_key __ipipe_smp_key;
+#define ipipe_smp_p (static_key_true(&__ipipe_smp_key))
+#endif /* SMP_ON_UP */
+#else /* !CONFIG_SMP */
+#define __ipipe_early_core_setup()	do { } while(0)
+#define __ipipe_hook_critical_ipi(ipd)	do { } while(0)
+#endif /* !CONFIG_SMP */
+#ifndef __ipipe_mach_init_platform
+#define __ipipe_mach_init_platform()	do { } while(0)
+#endif
+
+void __ipipe_enable_pipeline(void);
+
+void __ipipe_do_critical_sync(unsigned irq, void *cookie);
+
+void __ipipe_grab_irq(int irq, struct pt_regs *regs);
+
+void __ipipe_exit_irq(struct pt_regs *regs);
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+	return ffs(ul) - 1;
+}
+
+#define __ipipe_root_tick_p(regs) (!arch_irqs_disabled_flags(regs->ARM_cpsr))
+
+#ifdef CONFIG_IRQ_DOMAIN
+static inline
+int ipipe_handle_domain_irq(struct irq_domain *domain,
+			    unsigned int hwirq, struct pt_regs *regs)
+{
+	unsigned int irq;
+	irq = irq_find_mapping(domain, hwirq);
+	__ipipe_grab_irq(irq, regs);
+
+	return 0;
+}
+#endif /* irq domains */
+
+#else /* !CONFIG_IPIPE */
+
+#include <linux/irq.h>
+#include <linux/irqdesc.h>
+
+#define __ipipe_tsc_update()	do { } while(0)
+
+#define hard_smp_processor_id()		smp_processor_id()
+
+#ifdef CONFIG_SMP
+static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs)
+{
+	handle_IPI(irq, regs);
+}
+#endif /* CONFIG_SMP */
+
+struct timekeeper;
+static inline void __ipipe_update_vsyscall(struct timekeeper *tk) {}
+
+#endif /* !CONFIG_IPIPE */
+
+#endif	/* !__ARM_IPIPE_H */

arch/arm/include/asm/ipipe_base.h

+/* -*- linux-c -*-
+ * arch/arm/include/asm/ipipe_base.h
+ *
+ * Copyright (C) 2007 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ASM_ARM_IPIPE_BASE_H
+#define __ASM_ARM_IPIPE_BASE_H
+
+#include <asm-generic/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#ifdef CONFIG_SMP
+
+extern unsigned __ipipe_first_ipi;
+
+#define IPIPE_CRITICAL_IPI	__ipipe_first_ipi
+#define IPIPE_HRTIMER_IPI	(IPIPE_CRITICAL_IPI + 1)
+#define IPIPE_RESCHEDULE_IPI	(IPIPE_CRITICAL_IPI + 2)
+#define IPIPE_SERVICE_VNMI	(IPIPE_CRITICAL_IPI + 3)
+
+#define IPIPE_LAST_IPI		IPIPE_SERVICE_VNMI
+
+#define hard_smp_processor_id()	raw_smp_processor_id()
+
+#ifdef CONFIG_SMP_ON_UP
+unsigned __ipipe_processor_id(void);
+
+#define ipipe_processor_id()						\
+	({								\
+		register unsigned int cpunum __asm__ ("r0");		\
+		register unsigned int r1 __asm__ ("r1");		\
+		register unsigned int r2 __asm__ ("r2");		\
+		register unsigned int r3 __asm__ ("r3");		\
+		register unsigned int ip __asm__ ("ip");		\
+		register unsigned int lr __asm__ ("lr");		\
+		__asm__ __volatile__ ("\n"				\
+			"1:	bl __ipipe_processor_id\n"		\
+			"	.pushsection \".alt.smp.init\", \"a\"\n" \
+			"	.long	1b\n"				\
+			"	mov	%0, #0\n"			\
+			"	.popsection"				\
+				: "=r"(cpunum),	"=r"(r1), "=r"(r2), "=r"(r3), \
+				  "=r"(ip), "=r"(lr)			\
+				: /* */ : "cc");			\
+		cpunum;						\
+	})
+#else /* !SMP_ON_UP */
+#define ipipe_processor_id() raw_smp_processor_id()
+#endif /* !SMP_ON_UP */
+
+#define IPIPE_ARCH_HAVE_VIRQ_IPI
+
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id()  (0)
+#endif /* !CONFIG_IPIPE */
+
+#define IPIPE_TRAP_MAYDAY        0	/* Internal recovery trap */
+
+#endif /* CONFIG_IPIPE */
+
+#endif /* __ASM_ARM_IPIPE_BASE_H */

arch/arm/include/asm/ipipe_hwirq.h

+/* -*- linux-c -*-
+ * arch/arm/include/asm/ipipe_hwirq.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2005 Stelian Pop.
+ * Copyright (C) 2006-2008 Gilles Chanteperdrix.
+ * Copyright (C) 2010 Philippe Gerum (SMP port).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _ASM_ARM_IPIPE_HWIRQ_H
+#define _ASM_ARM_IPIPE_HWIRQ_H
+
+#include <asm-generic/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#include <linux/ipipe_trace.h>
+
+#define hard_local_irq_restore_notrace(x)				\
+	__asm__ __volatile__(						\
+	"msr	cpsr_c, %0		@ hard_local_irq_restore\n"	\
+	:								\
+	: "r" (x)							\
+	: "memory", "cc")
+
+static inline void hard_local_irq_disable_notrace(void)
+{
+#if __LINUX_ARM_ARCH__ >= 6
+	__asm__("cpsid i	@ __cli" : : : "memory", "cc");
+#else /* linux arch <= 5 */
+	unsigned long temp;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ hard_local_irq_disable\n"
+		"orr	%0, %0, #128\n"
+		"msr	cpsr_c, %0"
+		: "=r" (temp)
+		:
+		: "memory", "cc");
+#endif /* linux arch <= 5 */
+}
+
+static inline void hard_local_irq_enable_notrace(void)
+{
+#if __LINUX_ARM_ARCH__ >= 6
+	__asm__("cpsie i	@ __sti" : : : "memory", "cc");
+#else /* linux arch <= 5 */
+	unsigned long temp;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ hard_local_irq_enable\n"
+		"bic	%0, %0, #128\n"
+		"msr	cpsr_c, %0"
+		: "=r" (temp)
+		:
+		: "memory", "cc");
+#endif /* linux arch <= 5 */
+}
+
+static inline void hard_local_fiq_disable_notrace(void)
+{
+#if __LINUX_ARM_ARCH__ >= 6
+	__asm__("cpsid f	@ __clf" : : : "memory", "cc");
+#else /* linux arch <= 5 */
+	unsigned long temp;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ clf\n"
+		"orr	%0, %0, #64\n"
+		"msr	cpsr_c, %0"
+		: "=r" (temp)
+		:
+		: "memory", "cc");
+#endif /* linux arch <= 5 */
+}
+
+static inline void hard_local_fiq_enable_notrace(void)
+{
+#if __LINUX_ARM_ARCH__ >= 6
+	__asm__("cpsie f	@ __stf" : : : "memory", "cc");
+#else /* linux arch <= 5 */
+	unsigned long temp;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ stf\n"
+		"bic	%0, %0, #64\n"
+		"msr	cpsr_c, %0"
+		: "=r" (temp)
+		:
+		: "memory", "cc");
+#endif /* linux arch <= 5 */
+}
+
+static inline unsigned long hard_local_irq_save_notrace(void)
+{
+	unsigned long res;
+#if __LINUX_ARM_ARCH__ >= 6
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ hard_local_irq_save\n"
+		"cpsid	i"
+		: "=r" (res) : : "memory", "cc");
+#else /* linux arch <= 5 */
+	unsigned long temp;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ hard_local_irq_save\n"
+		"orr	%1, %0, #128\n"
+		"msr	cpsr_c, %1"
+		: "=r" (res), "=r" (temp)
+		:
+		: "memory", "cc");
+#endif /* linux arch <= 5 */
+	  return res;
+}
+
+static inline int arch_irqs_disabled_flags(unsigned long flags)
+{
+	return (int)((flags) & PSR_I_BIT);
+}
+
+static inline unsigned long hard_local_save_flags(void)
+{
+	unsigned long flags;
+	__asm__ __volatile__(
+		"mrs	%0, cpsr		@ hard_local_save_flags"
+		: "=r" (flags) : : "memory", "cc");
+	return flags;
+}
+
+#define hard_irqs_disabled_flags(flags) arch_irqs_disabled_flags(flags)
+
+static inline int hard_irqs_disabled(void)
+{
+	return hard_irqs_disabled_flags(hard_local_save_flags());
+}
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+static inline void hard_local_irq_disable(void)
+{
+	if (!hard_irqs_disabled()) {
+		hard_local_irq_disable_notrace();
+		ipipe_trace_begin(0x80000000);
+	}
+}
+
+static inline void hard_local_irq_enable(void)
+{
+	if (hard_irqs_disabled()) {
+		ipipe_trace_end(0x80000000);
+		hard_local_irq_enable_notrace();
+	}
+}
+
+static inline unsigned long hard_local_irq_save(void)
+{
+	unsigned long flags;
+
+	flags = hard_local_irq_save_notrace();
+	if (!arch_irqs_disabled_flags(flags))
+		ipipe_trace_begin(0x80000001);
+
+	return flags;
+}
+
+static inline void hard_local_irq_restore(unsigned long x)
+{
+	if (!arch_irqs_disabled_flags(x))
+		ipipe_trace_end(0x80000001);
+
+	hard_local_irq_restore_notrace(x);
+}
+
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define hard_local_irq_disable    hard_local_irq_disable_notrace
+#define hard_local_irq_enable     hard_local_irq_enable_notrace
+#define hard_local_irq_save       hard_local_irq_save_notrace
+#define hard_local_irq_restore    hard_local_irq_restore_notrace
+
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define arch_local_irq_disable()		\
+	({					\
+		ipipe_stall_root();		\
+		barrier();			\
+	})
+
+#define arch_local_irq_enable()				\
+	do {						\
+		barrier();				\
+		ipipe_unstall_root();			\
+	} while (0)
+
+#define local_fiq_enable() hard_local_fiq_enable_notrace()
+
+#define local_fiq_disable() hard_local_fiq_disable_notrace()
+
+#define arch_local_irq_restore(flags)			\
+	do {						\
+		if (!arch_irqs_disabled_flags(flags))	\
+			arch_local_irq_enable();	\
+	} while (0)
+
+#define arch_local_irq_save()						\
+	({								\
+		unsigned long _flags;					\
+		_flags = ipipe_test_and_stall_root() << 7;		\
+		barrier();						\
+		_flags;							\