ipipe.h 7.96 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
/* -*- linux-c -*-
 * arch/arm/include/asm/ipipe.h
 *
 * Copyright (C) 2002-2005 Philippe Gerum.
 * Copyright (C) 2005 Stelian Pop.
 * Copyright (C) 2006-2008 Gilles Chanteperdrix.
 * Copyright (C) 2010 Philippe Gerum (SMP port).
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
 * USA; either version 2 of the License, or (at your option) any later
 * version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */

#ifndef __ARM_IPIPE_H
#define __ARM_IPIPE_H

#ifdef CONFIG_IPIPE

#define BROKEN_BUILTIN_RETURN_ADDRESS
#undef __BUILTIN_RETURN_ADDRESS0
#undef __BUILTIN_RETURN_ADDRESS1
#ifdef CONFIG_FRAME_POINTER
34
35
36
#define __BUILTIN_RETURN_ADDRESS0 arm64_return_addr(0)
#define __BUILTIN_RETURN_ADDRESS1 arm64_return_addr(1)
extern unsigned long arm64_return_addr(int level);
37
38
39
40
41
42
43
#else
#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
#define __BUILTIN_RETURN_ADDRESS1 (0)
#endif

#include <linux/jump_label.h>
#include <linux/ipipe_trace.h>
44
#include <linux/ipipe_debug.h>
45

46
#define IPIPE_CORE_RELEASE	12
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

struct ipipe_domain;

#define IPIPE_TSC_TYPE_NONE			0
#define IPIPE_TSC_TYPE_FREERUNNING		1
#define IPIPE_TSC_TYPE_DECREMENTER		2
#define IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN	3
#define IPIPE_TSC_TYPE_FREERUNNING_TWICE	4
#define IPIPE_TSC_TYPE_FREERUNNING_ARCH		5

/* tscinfo, exported to user-space */
struct __ipipe_tscinfo {
	unsigned type;
	unsigned freq;
	unsigned long counter_vaddr;
	union {
		struct {
			unsigned long counter_paddr;
			unsigned long long mask;
		};
		struct {
			unsigned *counter; /* Hw counter physical address */
			unsigned long long mask; /* Significant bits in the hw counter. */
			unsigned long long *tsc; /* 64 bits tsc value. */
		} fr;
		struct {
			unsigned *counter; /* Hw counter physical address */
			unsigned long long mask; /* Significant bits in the hw counter. */
			unsigned *last_cnt; /* Counter value when updating
						tsc value. */
			unsigned long long *tsc; /* 64 bits tsc value. */
		} dec;
	} u;
};

struct ipipe_arch_sysinfo {
	struct __ipipe_tscinfo tsc;
};


/* arch specific stuff */
extern char __ipipe_tsc_area[];
void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info);

#ifdef CONFIG_IPIPE_ARM_KUSER_TSC
unsigned long long __ipipe_tsc_get(void) __attribute__((long_call));
void __ipipe_tsc_register(struct __ipipe_tscinfo *info);
void __ipipe_tsc_update(void);
extern unsigned long __ipipe_kuser_tsc_freq;
#define __ipipe_hrclock_freq __ipipe_kuser_tsc_freq
#else /* ! generic tsc */
98
static inline void __ipipe_mach_update_tsc(void) {}
99
100
unsigned long long __ipipe_mach_get_tsc(void);
#define __ipipe_tsc_get() __ipipe_mach_get_tsc()
101
102
void __ipipe_tsc_register(struct __ipipe_tscinfo *info);
static inline void __ipipe_tsc_update(void) {}
103
104
105
106
107
108
109
110
111
112
113
114
#ifndef __ipipe_hrclock_freq
extern unsigned long __ipipe_hrtimer_freq;
#define __ipipe_hrclock_freq __ipipe_hrtimer_freq
#endif /* !__ipipe_mach_hrclock_freq */
#endif /* ! generic tsc */

#ifdef CONFIG_IPIPE_DEBUG_INTERNAL
extern void (*__ipipe_mach_hrtimer_debug)(unsigned irq);
#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */

#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH

115
#define ipipe_mm_switch_protect(__flags)	\
116
	do {					\
117
		(void)(__flags);		\
118
119
	} while(0)

120
#define ipipe_mm_switch_unprotect(__flags)	\
121
	do {					\
122
		(void)(__flags);		\
123
124
125
126
	} while(0)

#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */

127
128
129
130
#define ipipe_mm_switch_protect(__flags)		\
	do {						\
		(__flags) = hard_cond_local_irq_save();	\
	} while (0)
131

132
133
#define ipipe_mm_switch_unprotect(__flags)	\
	hard_cond_local_irq_restore(__flags)
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */

#define ipipe_get_active_mm()	(__this_cpu_read(ipipe_percpu.active_mm))

#define ipipe_read_tsc(t)	do { t = __ipipe_tsc_get(); } while(0)
#define __ipipe_read_timebase()	__ipipe_tsc_get()

#define ipipe_tsc2ns(t) \
({ \
	unsigned long long delta = (t)*1000; \
	do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \
	(unsigned long)delta; \
})
#define ipipe_tsc2us(t) \
({ \
	unsigned long long delta = (t); \
	do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \
	(unsigned long)delta; \
})

static inline const char *ipipe_clock_name(void)
{
	return "ipipe_tsc";
}

/* Private interface -- Internal use only */

#define __ipipe_enable_irq(irq)		enable_irq(irq)
#define __ipipe_disable_irq(irq)	disable_irq(irq)

/* PIC muting */
struct ipipe_mach_pic_muter {
	void (*enable_irqdesc)(struct ipipe_domain *ipd, unsigned irq);
	void (*disable_irqdesc)(struct ipipe_domain *ipd, unsigned irq);
	void (*mute)(void);
	void (*unmute)(void);
};

extern struct ipipe_mach_pic_muter ipipe_pic_muter;

void ipipe_pic_muter_register(struct ipipe_mach_pic_muter *muter);

void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq);

void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq);

static inline void ipipe_mute_pic(void)
{
	if (ipipe_pic_muter.mute)
		ipipe_pic_muter.mute();
}

static inline void ipipe_unmute_pic(void)
{
	if (ipipe_pic_muter.unmute)
		ipipe_pic_muter.unmute();
}

#define ipipe_notify_root_preemption() do { } while(0)

#ifdef CONFIG_SMP
void __ipipe_early_core_setup(void);
void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
void __ipipe_root_localtimer(unsigned int irq, void *cookie);
void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg);
void __ipipe_do_vnmi(unsigned int irq, void *cookie);
void __ipipe_grab_ipi(unsigned svc, struct pt_regs *regs);
void __ipipe_ipis_alloc(void);
void __ipipe_ipis_request(void);

static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs)
{
	__ipipe_grab_ipi(irq, regs);
}

#ifdef CONFIG_SMP_ON_UP
extern struct static_key __ipipe_smp_key;
#define ipipe_smp_p (static_key_true(&__ipipe_smp_key))
#endif /* SMP_ON_UP */
#else /* !CONFIG_SMP */
#define __ipipe_early_core_setup()	do { } while(0)
#define __ipipe_hook_critical_ipi(ipd)	do { } while(0)
#endif /* !CONFIG_SMP */
#ifndef __ipipe_mach_init_platform
#define __ipipe_mach_init_platform()	do { } while(0)
#endif

void __ipipe_enable_pipeline(void);

void __ipipe_do_critical_sync(unsigned irq, void *cookie);

void __ipipe_grab_irq(int irq, struct pt_regs *regs);

void __ipipe_exit_irq(struct pt_regs *regs);

static inline void ipipe_handle_multi_irq(int irq, struct pt_regs *regs)
{
	__ipipe_grab_irq(irq, regs);
}

static inline unsigned long __ipipe_ffnz(unsigned long ul)
{
237
238
239
240
241
242
243
244
245
246
	int __r;

	/* zero input is not valid */
	IPIPE_WARN(ul == 0);

	__asm__ ("rbit\t%0, %1\n"
	         "clz\t%0, %0\n"
	        : "=r" (__r) : "r"(ul) : "cc");

	return __r;
247
248
249
250
251
}

#define __ipipe_syscall_watched_p(p, sc)				\
	(ipipe_notifier_enabled_p(p) || (unsigned long)sc >= __NR_syscalls)

252
#define __ipipe_root_tick_p(regs) (!arch_irqs_disabled_flags(regs->pstate))
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271

#else /* !CONFIG_IPIPE */

#define __ipipe_tsc_update()	do { } while(0)

#define hard_smp_processor_id()		smp_processor_id()

#define ipipe_mm_switch_protect(flags) \
	do {					\
		(void) (flags);			\
	} while(0)

#define ipipe_mm_switch_unprotect(flags)	\
	do {					\
		(void) (flags);			\
	} while(0)

static inline void ipipe_handle_multi_irq(int irq, struct pt_regs *regs)
{
272
	__handle_domain_irq(NULL, irq, false, regs);
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
}

#ifdef CONFIG_SMP
static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs)
{
	handle_IPI(irq, regs);
}
#endif /* CONFIG_SMP */
#endif /* CONFIG_IPIPE */

#if defined (CONFIG_IPIPE_DEBUG) &&		\
	(defined(CONFIG_DEBUG_LL) || defined(CONFIG_SERIAL_8250_CONSOLE))
void __ipipe_serial_debug(const char *fmt, ...);
#else
#define __ipipe_serial_debug(fmt, args...)	do { } while (0)
#endif

#endif	/* !__ARM_IPIPE_H */