Commit 68f7082f authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

perf/x86: Ensure perf_sched_cb_{inc,dec}() is only called from pmu::{add,del}()

Currently perf_sched_cb_{inc,dec}() are called from
pmu::{start,stop}(), which has the problem that this can happen from
NMI context, this is making it hard to optimize perf_pmu_sched_task().

Furthermore, we really only need this accounting on pmu::{add,del}(),
so doing it from pmu::{start,stop}() is doing more work than we really
need.

Introduce x86_pmu::{add,del}() and wire up the LBR and PEBS.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Stephane Eranian <eranian@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Vince Weaver <vincent.weaver@maine.edu>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 09e61b4f
...@@ -1201,6 +1201,9 @@ static int x86_pmu_add(struct perf_event *event, int flags) ...@@ -1201,6 +1201,9 @@ static int x86_pmu_add(struct perf_event *event, int flags)
* If group events scheduling transaction was started, * If group events scheduling transaction was started,
* skip the schedulability test here, it will be performed * skip the schedulability test here, it will be performed
* at commit time (->commit_txn) as a whole. * at commit time (->commit_txn) as a whole.
*
* If commit fails, we'll call ->del() on all events
* for which ->add() was called.
*/ */
if (cpuc->txn_flags & PERF_PMU_TXN_ADD) if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
goto done_collect; goto done_collect;
...@@ -1223,6 +1226,14 @@ static int x86_pmu_add(struct perf_event *event, int flags) ...@@ -1223,6 +1226,14 @@ static int x86_pmu_add(struct perf_event *event, int flags)
cpuc->n_added += n - n0; cpuc->n_added += n - n0;
cpuc->n_txn += n - n0; cpuc->n_txn += n - n0;
if (x86_pmu.add) {
/*
* This is before x86_pmu_enable() will call x86_pmu_start(),
* so we enable LBRs before an event needs them etc..
*/
x86_pmu.add(event);
}
ret = 0; ret = 0;
out: out:
return ret; return ret;
...@@ -1346,7 +1357,7 @@ static void x86_pmu_del(struct perf_event *event, int flags) ...@@ -1346,7 +1357,7 @@ static void x86_pmu_del(struct perf_event *event, int flags)
event->hw.flags &= ~PERF_X86_EVENT_COMMITTED; event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
/* /*
* If we're called during a txn, we don't need to do anything. * If we're called during a txn, we only need to undo x86_pmu.add.
* The events never got scheduled and ->cancel_txn will truncate * The events never got scheduled and ->cancel_txn will truncate
* the event_list. * the event_list.
* *
...@@ -1354,7 +1365,7 @@ static void x86_pmu_del(struct perf_event *event, int flags) ...@@ -1354,7 +1365,7 @@ static void x86_pmu_del(struct perf_event *event, int flags)
* an event added during that same TXN. * an event added during that same TXN.
*/ */
if (cpuc->txn_flags & PERF_PMU_TXN_ADD) if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
return; goto do_del;
/* /*
* Not a TXN, therefore cleanup properly. * Not a TXN, therefore cleanup properly.
...@@ -1384,6 +1395,15 @@ static void x86_pmu_del(struct perf_event *event, int flags) ...@@ -1384,6 +1395,15 @@ static void x86_pmu_del(struct perf_event *event, int flags)
--cpuc->n_events; --cpuc->n_events;
perf_event_update_userpage(event); perf_event_update_userpage(event);
do_del:
if (x86_pmu.del) {
/*
* This is after x86_pmu_stop(); so we disable LBRs after any
* event can need them etc..
*/
x86_pmu.del(event);
}
} }
int x86_pmu_handle_irq(struct pt_regs *regs) int x86_pmu_handle_irq(struct pt_regs *regs)
......
...@@ -1907,13 +1907,6 @@ static void intel_pmu_disable_event(struct perf_event *event) ...@@ -1907,13 +1907,6 @@ static void intel_pmu_disable_event(struct perf_event *event)
cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx); cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
cpuc->intel_cp_status &= ~(1ull << hwc->idx); cpuc->intel_cp_status &= ~(1ull << hwc->idx);
/*
* must disable before any actual event
* because any event may be combined with LBR
*/
if (needs_branch_stack(event))
intel_pmu_lbr_disable(event);
if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) { if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
intel_pmu_disable_fixed(hwc); intel_pmu_disable_fixed(hwc);
return; return;
...@@ -1925,6 +1918,14 @@ static void intel_pmu_disable_event(struct perf_event *event) ...@@ -1925,6 +1918,14 @@ static void intel_pmu_disable_event(struct perf_event *event)
intel_pmu_pebs_disable(event); intel_pmu_pebs_disable(event);
} }
static void intel_pmu_del_event(struct perf_event *event)
{
if (needs_branch_stack(event))
intel_pmu_lbr_del(event);
if (event->attr.precise_ip)
intel_pmu_pebs_del(event);
}
static void intel_pmu_enable_fixed(struct hw_perf_event *hwc) static void intel_pmu_enable_fixed(struct hw_perf_event *hwc)
{ {
int idx = hwc->idx - INTEL_PMC_IDX_FIXED; int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
...@@ -1968,12 +1969,6 @@ static void intel_pmu_enable_event(struct perf_event *event) ...@@ -1968,12 +1969,6 @@ static void intel_pmu_enable_event(struct perf_event *event)
intel_pmu_enable_bts(hwc->config); intel_pmu_enable_bts(hwc->config);
return; return;
} }
/*
* must enabled before any actual event
* because any event may be combined with LBR
*/
if (needs_branch_stack(event))
intel_pmu_lbr_enable(event);
if (event->attr.exclude_host) if (event->attr.exclude_host)
cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx); cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);
...@@ -1994,6 +1989,14 @@ static void intel_pmu_enable_event(struct perf_event *event) ...@@ -1994,6 +1989,14 @@ static void intel_pmu_enable_event(struct perf_event *event)
__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE); __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
} }
static void intel_pmu_add_event(struct perf_event *event)
{
if (event->attr.precise_ip)
intel_pmu_pebs_add(event);
if (needs_branch_stack(event))
intel_pmu_lbr_add(event);
}
/* /*
* Save and restart an expired event. Called by NMI contexts, * Save and restart an expired event. Called by NMI contexts,
* so it has to be careful about preempting normal event ops: * so it has to be careful about preempting normal event ops:
...@@ -3290,6 +3293,8 @@ static __initconst const struct x86_pmu intel_pmu = { ...@@ -3290,6 +3293,8 @@ static __initconst const struct x86_pmu intel_pmu = {
.enable_all = intel_pmu_enable_all, .enable_all = intel_pmu_enable_all,
.enable = intel_pmu_enable_event, .enable = intel_pmu_enable_event,
.disable = intel_pmu_disable_event, .disable = intel_pmu_disable_event,
.add = intel_pmu_add_event,
.del = intel_pmu_del_event,
.hw_config = intel_pmu_hw_config, .hw_config = intel_pmu_hw_config,
.schedule_events = x86_schedule_events, .schedule_events = x86_schedule_events,
.eventsel = MSR_ARCH_PERFMON_EVENTSEL0, .eventsel = MSR_ARCH_PERFMON_EVENTSEL0,
......
...@@ -844,7 +844,7 @@ pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc, struct pmu *pmu) ...@@ -844,7 +844,7 @@ pebs_update_state(bool needed_cb, struct cpu_hw_events *cpuc, struct pmu *pmu)
} }
} }
static void intel_pmu_pebs_add(struct perf_event *event) void intel_pmu_pebs_add(struct perf_event *event)
{ {
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
...@@ -863,8 +863,6 @@ void intel_pmu_pebs_enable(struct perf_event *event) ...@@ -863,8 +863,6 @@ void intel_pmu_pebs_enable(struct perf_event *event)
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
struct debug_store *ds = cpuc->ds; struct debug_store *ds = cpuc->ds;
intel_pmu_pebs_add(event);
hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT; hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
cpuc->pebs_enabled |= 1ULL << hwc->idx; cpuc->pebs_enabled |= 1ULL << hwc->idx;
...@@ -884,7 +882,7 @@ void intel_pmu_pebs_enable(struct perf_event *event) ...@@ -884,7 +882,7 @@ void intel_pmu_pebs_enable(struct perf_event *event)
} }
} }
static void intel_pmu_pebs_del(struct perf_event *event) void intel_pmu_pebs_del(struct perf_event *event)
{ {
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct hw_perf_event *hwc = &event->hw; struct hw_perf_event *hwc = &event->hw;
...@@ -916,8 +914,6 @@ void intel_pmu_pebs_disable(struct perf_event *event) ...@@ -916,8 +914,6 @@ void intel_pmu_pebs_disable(struct perf_event *event)
wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled); wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
hwc->config |= ARCH_PERFMON_EVENTSEL_INT; hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
intel_pmu_pebs_del(event);
} }
void intel_pmu_pebs_enable_all(void) void intel_pmu_pebs_enable_all(void)
......
...@@ -422,7 +422,7 @@ static inline bool branch_user_callstack(unsigned br_sel) ...@@ -422,7 +422,7 @@ static inline bool branch_user_callstack(unsigned br_sel)
return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK); return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK);
} }
void intel_pmu_lbr_enable(struct perf_event *event) void intel_pmu_lbr_add(struct perf_event *event)
{ {
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct x86_perf_task_context *task_ctx; struct x86_perf_task_context *task_ctx;
...@@ -450,7 +450,7 @@ void intel_pmu_lbr_enable(struct perf_event *event) ...@@ -450,7 +450,7 @@ void intel_pmu_lbr_enable(struct perf_event *event)
perf_sched_cb_inc(event->ctx->pmu); perf_sched_cb_inc(event->ctx->pmu);
} }
void intel_pmu_lbr_disable(struct perf_event *event) void intel_pmu_lbr_del(struct perf_event *event)
{ {
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct x86_perf_task_context *task_ctx; struct x86_perf_task_context *task_ctx;
......
...@@ -510,6 +510,8 @@ struct x86_pmu { ...@@ -510,6 +510,8 @@ struct x86_pmu {
void (*enable_all)(int added); void (*enable_all)(int added);
void (*enable)(struct perf_event *); void (*enable)(struct perf_event *);
void (*disable)(struct perf_event *); void (*disable)(struct perf_event *);
void (*add)(struct perf_event *);
void (*del)(struct perf_event *);
int (*hw_config)(struct perf_event *event); int (*hw_config)(struct perf_event *event);
int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign); int (*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
unsigned eventsel; unsigned eventsel;
...@@ -890,6 +892,10 @@ extern struct event_constraint intel_skl_pebs_event_constraints[]; ...@@ -890,6 +892,10 @@ extern struct event_constraint intel_skl_pebs_event_constraints[];
struct event_constraint *intel_pebs_constraints(struct perf_event *event); struct event_constraint *intel_pebs_constraints(struct perf_event *event);
void intel_pmu_pebs_add(struct perf_event *event);
void intel_pmu_pebs_del(struct perf_event *event);
void intel_pmu_pebs_enable(struct perf_event *event); void intel_pmu_pebs_enable(struct perf_event *event);
void intel_pmu_pebs_disable(struct perf_event *event); void intel_pmu_pebs_disable(struct perf_event *event);
...@@ -908,9 +914,9 @@ u64 lbr_from_signext_quirk_wr(u64 val); ...@@ -908,9 +914,9 @@ u64 lbr_from_signext_quirk_wr(u64 val);
void intel_pmu_lbr_reset(void); void intel_pmu_lbr_reset(void);
void intel_pmu_lbr_enable(struct perf_event *event); void intel_pmu_lbr_add(struct perf_event *event);
void intel_pmu_lbr_disable(struct perf_event *event); void intel_pmu_lbr_del(struct perf_event *event);
void intel_pmu_lbr_enable_all(bool pmi); void intel_pmu_lbr_enable_all(bool pmi);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment