Commit 31bbb9b5 authored by Linus Torvalds's avatar Linus Torvalds
Browse files

Merge branch 'timers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'timers-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  itimers: Add tracepoints for itimer
  hrtimer: Add tracepoint for hrtimers
  timers: Add tracepoints for timer_list timers
  cputime: Optimize jiffies_to_cputime(1)
  itimers: Simplify arm_timer() code a bit
  itimers: Fix periodic tics precision
  itimers: Merge ITIMER_VIRT and ITIMER_PROF

Trivial header file include conflicts in kernel/fork.c
parents ff830b8e 3f0a525e
......@@ -30,6 +30,7 @@ typedef u64 cputime_t;
typedef u64 cputime64_t;
#define cputime_zero ((cputime_t)0)
#define cputime_one_jiffy jiffies_to_cputime(1)
#define cputime_max ((~((cputime_t)0) >> 1) - 1)
#define cputime_add(__a, __b) ((__a) + (__b))
#define cputime_sub(__a, __b) ((__a) - (__b))
......
......@@ -18,6 +18,9 @@
#ifndef CONFIG_VIRT_CPU_ACCOUNTING
#include <asm-generic/cputime.h>
#ifdef __KERNEL__
static inline void setup_cputime_one_jiffy(void) { }
#endif
#else
#include <linux/types.h>
......@@ -48,6 +51,11 @@ typedef u64 cputime64_t;
#ifdef __KERNEL__
/*
* One jiffy in timebase units computed during initialization
*/
extern cputime_t cputime_one_jiffy;
/*
* Convert cputime <-> jiffies
*/
......@@ -89,6 +97,11 @@ static inline cputime_t jiffies_to_cputime(const unsigned long jif)
return ct;
}
static inline void setup_cputime_one_jiffy(void)
{
cputime_one_jiffy = jiffies_to_cputime(1);
}
static inline cputime64_t jiffies64_to_cputime64(const u64 jif)
{
cputime_t ct;
......
......@@ -193,6 +193,8 @@ EXPORT_SYMBOL(__cputime_clockt_factor);
DEFINE_PER_CPU(unsigned long, cputime_last_delta);
DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta);
cputime_t cputime_one_jiffy;
static void calc_cputime_factors(void)
{
struct div_result res;
......@@ -501,6 +503,7 @@ static int __init iSeries_tb_recal(void)
tb_to_xs = divres.result_low;
vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
vdso_data->tb_to_xs = tb_to_xs;
setup_cputime_one_jiffy();
}
else {
printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"
......@@ -960,6 +963,7 @@ void __init time_init(void)
tb_ticks_per_usec = ppc_tb_freq / 1000000;
tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);
calc_cputime_factors();
setup_cputime_one_jiffy();
/*
* Calculate the length of each tick in ns. It will not be
......
......@@ -42,6 +42,7 @@ __div(unsigned long long n, unsigned int base)
#endif /* __s390x__ */
#define cputime_zero (0ULL)
#define cputime_one_jiffy jiffies_to_cputime(1)
#define cputime_max ((~0UL >> 1) - 1)
#define cputime_add(__a, __b) ((__a) + (__b))
#define cputime_sub(__a, __b) ((__a) - (__b))
......
......@@ -7,6 +7,7 @@
typedef unsigned long cputime_t;
#define cputime_zero (0UL)
#define cputime_one_jiffy jiffies_to_cputime(1)
#define cputime_max ((~0UL >> 1) - 1)
#define cputime_add(__a, __b) ((__a) + (__b))
#define cputime_sub(__a, __b) ((__a) - (__b))
......
......@@ -493,6 +493,13 @@ struct pacct_struct {
unsigned long ac_minflt, ac_majflt;
};
struct cpu_itimer {
cputime_t expires;
cputime_t incr;
u32 error;
u32 incr_error;
};
/**
* struct task_cputime - collected CPU time counts
* @utime: time spent in user mode, in &cputime_t units
......@@ -587,9 +594,12 @@ struct signal_struct {
struct pid *leader_pid;
ktime_t it_real_incr;
/* ITIMER_PROF and ITIMER_VIRTUAL timers for the process */
cputime_t it_prof_expires, it_virt_expires;
cputime_t it_prof_incr, it_virt_incr;
/*
* ITIMER_PROF and ITIMER_VIRTUAL timers for the process, we use
* CPUCLOCK_PROF and CPUCLOCK_VIRT for indexing array as these
* values are defined to 0 and 1 respectively
*/
struct cpu_itimer it[2];
/*
* Thread group totals for process CPU timers.
......
#undef TRACE_SYSTEM
#define TRACE_SYSTEM timer
#if !defined(_TRACE_TIMER_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_TIMER_H
#include <linux/tracepoint.h>
#include <linux/hrtimer.h>
#include <linux/timer.h>
/**
* timer_init - called when the timer is initialized
* @timer: pointer to struct timer_list
*/
TRACE_EVENT(timer_init,
TP_PROTO(struct timer_list *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
),
TP_fast_assign(
__entry->timer = timer;
),
TP_printk("timer %p", __entry->timer)
);
/**
* timer_start - called when the timer is started
* @timer: pointer to struct timer_list
* @expires: the timers expiry time
*/
TRACE_EVENT(timer_start,
TP_PROTO(struct timer_list *timer, unsigned long expires),
TP_ARGS(timer, expires),
TP_STRUCT__entry(
__field( void *, timer )
__field( void *, function )
__field( unsigned long, expires )
__field( unsigned long, now )
),
TP_fast_assign(
__entry->timer = timer;
__entry->function = timer->function;
__entry->expires = expires;
__entry->now = jiffies;
),
TP_printk("timer %p: func %pf, expires %lu, timeout %ld",
__entry->timer, __entry->function, __entry->expires,
(long)__entry->expires - __entry->now)
);
/**
* timer_expire_entry - called immediately before the timer callback
* @timer: pointer to struct timer_list
*
* Allows to determine the timer latency.
*/
TRACE_EVENT(timer_expire_entry,
TP_PROTO(struct timer_list *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
__field( unsigned long, now )
),
TP_fast_assign(
__entry->timer = timer;
__entry->now = jiffies;
),
TP_printk("timer %p: now %lu", __entry->timer, __entry->now)
);
/**
* timer_expire_exit - called immediately after the timer callback returns
* @timer: pointer to struct timer_list
*
* When used in combination with the timer_expire_entry tracepoint we can
* determine the runtime of the timer callback function.
*
* NOTE: Do NOT derefernce timer in TP_fast_assign. The pointer might
* be invalid. We solely track the pointer.
*/
TRACE_EVENT(timer_expire_exit,
TP_PROTO(struct timer_list *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field(void *, timer )
),
TP_fast_assign(
__entry->timer = timer;
),
TP_printk("timer %p", __entry->timer)
);
/**
* timer_cancel - called when the timer is canceled
* @timer: pointer to struct timer_list
*/
TRACE_EVENT(timer_cancel,
TP_PROTO(struct timer_list *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
),
TP_fast_assign(
__entry->timer = timer;
),
TP_printk("timer %p", __entry->timer)
);
/**
* hrtimer_init - called when the hrtimer is initialized
* @timer: pointer to struct hrtimer
* @clockid: the hrtimers clock
* @mode: the hrtimers mode
*/
TRACE_EVENT(hrtimer_init,
TP_PROTO(struct hrtimer *timer, clockid_t clockid,
enum hrtimer_mode mode),
TP_ARGS(timer, clockid, mode),
TP_STRUCT__entry(
__field( void *, timer )
__field( clockid_t, clockid )
__field( enum hrtimer_mode, mode )
),
TP_fast_assign(
__entry->timer = timer;
__entry->clockid = clockid;
__entry->mode = mode;
),
TP_printk("hrtimer %p, clockid %s, mode %s", __entry->timer,
__entry->clockid == CLOCK_REALTIME ?
"CLOCK_REALTIME" : "CLOCK_MONOTONIC",
__entry->mode == HRTIMER_MODE_ABS ?
"HRTIMER_MODE_ABS" : "HRTIMER_MODE_REL")
);
/**
* hrtimer_start - called when the hrtimer is started
* @timer: pointer to struct hrtimer
*/
TRACE_EVENT(hrtimer_start,
TP_PROTO(struct hrtimer *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
__field( void *, function )
__field( s64, expires )
__field( s64, softexpires )
),
TP_fast_assign(
__entry->timer = timer;
__entry->function = timer->function;
__entry->expires = hrtimer_get_expires(timer).tv64;
__entry->softexpires = hrtimer_get_softexpires(timer).tv64;
),
TP_printk("hrtimer %p, func %pf, expires %llu, softexpires %llu",
__entry->timer, __entry->function,
(unsigned long long)ktime_to_ns((ktime_t) {
.tv64 = __entry->expires }),
(unsigned long long)ktime_to_ns((ktime_t) {
.tv64 = __entry->softexpires }))
);
/**
* htimmer_expire_entry - called immediately before the hrtimer callback
* @timer: pointer to struct hrtimer
* @now: pointer to variable which contains current time of the
* timers base.
*
* Allows to determine the timer latency.
*/
TRACE_EVENT(hrtimer_expire_entry,
TP_PROTO(struct hrtimer *timer, ktime_t *now),
TP_ARGS(timer, now),
TP_STRUCT__entry(
__field( void *, timer )
__field( s64, now )
),
TP_fast_assign(
__entry->timer = timer;
__entry->now = now->tv64;
),
TP_printk("hrtimer %p, now %llu", __entry->timer,
(unsigned long long)ktime_to_ns((ktime_t) {
.tv64 = __entry->now }))
);
/**
* hrtimer_expire_exit - called immediately after the hrtimer callback returns
* @timer: pointer to struct hrtimer
*
* When used in combination with the hrtimer_expire_entry tracepoint we can
* determine the runtime of the callback function.
*/
TRACE_EVENT(hrtimer_expire_exit,
TP_PROTO(struct hrtimer *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
),
TP_fast_assign(
__entry->timer = timer;
),
TP_printk("hrtimer %p", __entry->timer)
);
/**
* hrtimer_cancel - called when the hrtimer is canceled
* @timer: pointer to struct hrtimer
*/
TRACE_EVENT(hrtimer_cancel,
TP_PROTO(struct hrtimer *timer),
TP_ARGS(timer),
TP_STRUCT__entry(
__field( void *, timer )
),
TP_fast_assign(
__entry->timer = timer;
),
TP_printk("hrtimer %p", __entry->timer)
);
/**
* itimer_state - called when itimer is started or canceled
* @which: name of the interval timer
* @value: the itimers value, itimer is canceled if value->it_value is
* zero, otherwise it is started
* @expires: the itimers expiry time
*/
TRACE_EVENT(itimer_state,
TP_PROTO(int which, const struct itimerval *const value,
cputime_t expires),
TP_ARGS(which, value, expires),
TP_STRUCT__entry(
__field( int, which )
__field( cputime_t, expires )
__field( long, value_sec )
__field( long, value_usec )
__field( long, interval_sec )
__field( long, interval_usec )
),
TP_fast_assign(
__entry->which = which;
__entry->expires = expires;
__entry->value_sec = value->it_value.tv_sec;
__entry->value_usec = value->it_value.tv_usec;
__entry->interval_sec = value->it_interval.tv_sec;
__entry->interval_usec = value->it_interval.tv_usec;
),
TP_printk("which %d, expires %lu, it_value %lu.%lu, it_interval %lu.%lu",
__entry->which, __entry->expires,
__entry->value_sec, __entry->value_usec,
__entry->interval_sec, __entry->interval_usec)
);
/**
* itimer_expire - called when itimer expires
* @which: type of the interval timer
* @pid: pid of the process which owns the timer
* @now: current time, used to calculate the latency of itimer
*/
TRACE_EVENT(itimer_expire,
TP_PROTO(int which, struct pid *pid, cputime_t now),
TP_ARGS(which, pid, now),
TP_STRUCT__entry(
__field( int , which )
__field( pid_t, pid )
__field( cputime_t, now )
),
TP_fast_assign(
__entry->which = which;
__entry->now = now;
__entry->pid = pid_nr(pid);
),
TP_printk("which %d, pid %d, now %lu", __entry->which,
(int) __entry->pid, __entry->now)
);
#endif /* _TRACE_TIMER_H */
/* This part must be outside protection */
#include <trace/define_trace.h>
......@@ -63,6 +63,7 @@
#include <linux/fs_struct.h>
#include <linux/magic.h>
#include <linux/perf_event.h>
#include <linux/posix-timers.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
......@@ -805,10 +806,10 @@ static void posix_cpu_timers_init_group(struct signal_struct *sig)
thread_group_cputime_init(sig);
/* Expiration times and increments. */
sig->it_virt_expires = cputime_zero;
sig->it_virt_incr = cputime_zero;
sig->it_prof_expires = cputime_zero;
sig->it_prof_incr = cputime_zero;
sig->it[CPUCLOCK_PROF].expires = cputime_zero;
sig->it[CPUCLOCK_PROF].incr = cputime_zero;
sig->it[CPUCLOCK_VIRT].expires = cputime_zero;
sig->it[CPUCLOCK_VIRT].incr = cputime_zero;
/* Cached expiration times. */
sig->cputime_expires.prof_exp = cputime_zero;
......
......@@ -48,6 +48,8 @@
#include <asm/uaccess.h>
#include <trace/events/timer.h>
/*
* The timer bases:
*
......@@ -442,6 +444,26 @@ static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
#endif
static inline void
debug_init(struct hrtimer *timer, clockid_t clockid,
enum hrtimer_mode mode)
{
debug_hrtimer_init(timer);
trace_hrtimer_init(timer, clockid, mode);
}
static inline void debug_activate(struct hrtimer *timer)
{
debug_hrtimer_activate(timer);
trace_hrtimer_start(timer);
}
static inline void debug_deactivate(struct hrtimer *timer)
{
debug_hrtimer_deactivate(timer);
trace_hrtimer_cancel(timer);
}
/* High resolution timer related functions */
#ifdef CONFIG_HIGH_RES_TIMERS
......@@ -798,7 +820,7 @@ static int enqueue_hrtimer(struct hrtimer *timer,
struct hrtimer *entry;
int leftmost = 1;
debug_hrtimer_activate(timer);
debug_activate(timer);
/*
* Find the right place in the rbtree:
......@@ -884,7 +906,7 @@ remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)
* reprogramming happens in the interrupt handler. This is a
* rare case and less expensive than a smp call.
*/
debug_hrtimer_deactivate(timer);
debug_deactivate(timer);
timer_stats_hrtimer_clear_start_info(timer);
reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);
__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE,
......@@ -1117,7 +1139,7 @@ static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
enum hrtimer_mode mode)
{
debug_hrtimer_init(timer);
debug_init(timer, clock_id, mode);
__hrtimer_init(timer, clock_id, mode);
}
EXPORT_SYMBOL_GPL(hrtimer_init);
......@@ -1141,7 +1163,7 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
}
EXPORT_SYMBOL_GPL(hrtimer_get_res);
static void __run_hrtimer(struct hrtimer *timer)
static void __run_hrtimer(struct hrtimer *timer, ktime_t *now)
{
struct hrtimer_clock_base *base = timer->base;
struct hrtimer_cpu_base *cpu_base = base->cpu_base;
......@@ -1150,7 +1172,7 @@ static void __run_hrtimer(struct hrtimer *timer)
WARN_ON(!irqs_disabled());
debug_hrtimer_deactivate(timer);
debug_deactivate(timer);
__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
timer_stats_account_hrtimer(timer);
fn = timer->function;
......@@ -1161,7 +1183,9 @@ static void __run_hrtimer(struct hrtimer *timer)
* the timer base.
*/
spin_unlock(&cpu_base->lock);
trace_hrtimer_expire_entry(timer, now);
restart = fn(timer);
trace_hrtimer_expire_exit(timer);
spin_lock(&cpu_base->lock);
/*
......@@ -1272,7 +1296,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
break;
}
__run_hrtimer(timer);
__run_hrtimer(timer, &basenow);
}
base++;
}
......@@ -1394,7 +1418,7 @@ void hrtimer_run_queues(void)
hrtimer_get_expires_tv64(timer))
break;
__run_hrtimer(timer);
__run_hrtimer(timer, &base->softirq_time);
}
spin_unlock(&cpu_base->lock);
}
......@@ -1571,7 +1595,7 @@ static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
while ((node = rb_first(&old_base->active))) {
timer = rb_entry(node, struct hrtimer, node);
BUG_ON(hrtimer_callback_running(timer));
debug_hrtimer_deactivate(timer);
debug_deactivate(timer);
/*
* Mark it as STATE_MIGRATE not INACTIVE otherwise the
......
......@@ -12,6 +12,7 @@
#include <linux/time.h>
#include <linux/posix-timers.h>
#include <linux/hrtimer.h>
#include <trace/events/timer.h>
#include <asm/uaccess.h>
......@@ -41,10 +42,43 @@ static struct timeval itimer_get_remtime(struct hrtimer *timer)
return ktime_to_timeval(rem);
}
static void get_cpu_itimer(struct task_struct *tsk, unsigned int clock_id,
struct itimerval *const value)
{
cputime_t cval, cinterval;
struct cpu_itimer *it = &tsk->signal->it[clock_id];
spin_lock_irq(&tsk->sighand->siglock);
cval = it->expires;
cinterval = it->incr;
if (!cputime_eq(cval, cputime_zero)) {
struct task_cputime cputime;
cputime_t t;
thread_group_cputimer(tsk, &cputime);
if (clock_id == CPUCLOCK_PROF)
t = cputime_add(cputime.utime, cputime.stime);
else
/* CPUCLOCK_VIRT */
t = cputime.utime;
if (cputime_le(cval, t))
/* about to fire */
cval = cputime_one_jiffy;
else
cval = cputime_sub(cval, t);
}
spin_unlock_irq(&tsk->sighand->siglock);
cputime_to_timeval(cval, &value->it_value);
cputime_to_timeval(cinterval, &value->it_interval);