Commit 963936d3 authored by Philippe Gerum's avatar Philippe Gerum Committed by Jan Kiszka
Browse files

ipipe: add 4th mapping level to interrupt log



Some configurations may define more than 256K distinct interrupts
(e.g. CONFIG_MAXSMP/x86), which is the limit for the current 3-level
mapping used for logging IRQs. Add a 4th mapping level to support
configurations up to 16M interrupts.
Signed-off-by: Jan Kiszka's avatarJan Kiszka <jan.kiszka@siemens.com>
parent 48f66bb3
......@@ -43,20 +43,25 @@ struct ipipe_vm_notifier;
/* Total number of IRQ slots */
#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE+IPIPE_NR_VIRQS)
#define IPIPE_IRQ_LOMAPSZ (IPIPE_NR_IRQS / BITS_PER_LONG)
#if IPIPE_IRQ_LOMAPSZ > BITS_PER_LONG
#define IPIPE_IRQ_MAPSZ (IPIPE_NR_IRQS / BITS_PER_LONG)
#define IPIPE_IRQ_1MAPSZ BITS_PER_LONG
#if IPIPE_IRQ_MAPSZ > BITS_PER_LONG * BITS_PER_LONG
/*
* We need a 3-level mapping. This allows us to handle up to 32k IRQ
* vectors on 32bit machines, 256k on 64bit ones.
* We need a 4-level mapping, up to 16M IRQs (64bit long, MAXSMP
* defines 512K IRQs).
*/
#define __IPIPE_3LEVEL_IRQMAP 1
#define IPIPE_IRQ_MDMAPSZ (__bpl_up(IPIPE_IRQ_LOMAPSZ) / BITS_PER_LONG)
#define __IPIPE_IRQMAP_LEVELS 4
#define IPIPE_IRQ_2MAPSZ (BITS_PER_LONG * BITS_PER_LONG)
#elif IPIPE_IRQ_MAPSZ > BITS_PER_LONG
/*
* 3-level mapping. Up to 256K IRQs (64 bit long).
*/
#define __IPIPE_IRQMAP_LEVELS 3
#else
/*
* 2-level mapping is enough. This allows us to handle up to 1024 IRQ
* vectors on 32bit machines, 4096 on 64bit ones.
* 2-level mapping is enough. Up to 4K IRQs (64 bit long).
*/
#define __IPIPE_2LEVEL_IRQMAP 1
#define __IPIPE_IRQMAP_LEVELS 2
#endif
/* Per-cpu pipeline status */
......@@ -132,12 +137,15 @@ extern struct ipipe_domain *ipipe_head_domain;
struct ipipe_percpu_domain_data {
unsigned long status; /* <= Must be first in struct. */
unsigned long irqpend_himap;
#ifdef __IPIPE_3LEVEL_IRQMAP
unsigned long irqpend_mdmap[IPIPE_IRQ_MDMAPSZ];
unsigned long irqpend_0map;
#if __IPIPE_IRQMAP_LEVELS >= 3
unsigned long irqpend_1map[IPIPE_IRQ_1MAPSZ];
#if __IPIPE_IRQMAP_LEVELS >= 4
unsigned long irqpend_2map[IPIPE_IRQ_2MAPSZ];
#endif
#endif
unsigned long irqpend_lomap[IPIPE_IRQ_LOMAPSZ];
unsigned long irqheld_map[IPIPE_IRQ_LOMAPSZ];
unsigned long irqpend_map[IPIPE_IRQ_MAPSZ];
unsigned long irqheld_map[IPIPE_IRQ_MAPSZ];
unsigned long irqall[IPIPE_NR_IRQS];
struct ipipe_domain *domain;
int coflags;
......@@ -346,7 +354,7 @@ extern unsigned long __ipipe_root_status;
*/
static inline int __ipipe_ipending_p(struct ipipe_percpu_domain_data *pd)
{
return pd->irqpend_himap != 0;
return pd->irqpend_0map != 0;
}
static inline unsigned long
......
......@@ -605,8 +605,6 @@ void __ipipe_spin_unlock_irqcomplete(unsigned long x)
hard_local_irq_restore(x);
}
#ifdef __IPIPE_3LEVEL_IRQMAP
/* Must be called hw IRQs off. */
static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
unsigned int irq)
......@@ -615,6 +613,147 @@ static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
p->irqall[irq]++;
}
#if __IPIPE_IRQMAP_LEVELS == 4
/* Must be called hw IRQs off. */
void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
{
struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd);
int l0b, l1b, l2b;
IPIPE_WARN_ONCE(!hard_irqs_disabled());
l0b = irq / (BITS_PER_LONG * BITS_PER_LONG * BITS_PER_LONG);
l1b = irq / (BITS_PER_LONG * BITS_PER_LONG);
l2b = irq / BITS_PER_LONG;
if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
__set_bit(l0b, &p->irqpend_0map);
__set_bit(l1b, p->irqpend_1map);
__set_bit(l2b, p->irqpend_2map);
__set_bit(irq, p->irqpend_map);
} else
__set_bit(irq, p->irqheld_map);
p->irqall[irq]++;
}
EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending);
/* Must be called hw IRQs off. */
void __ipipe_lock_irq(unsigned int irq)
{
struct ipipe_domain *ipd = ipipe_root_domain;
struct ipipe_percpu_domain_data *p;
int l0b, l1b, l2b;
IPIPE_WARN_ONCE(!hard_irqs_disabled());
/*
* Interrupts requested by a registered head domain cannot be
* locked, since this would make no sense: interrupts are
* globally masked at CPU level when the head domain is
* stalled, so there is no way we could encounter the
* situation IRQ locks are handling.
*/
if (test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
return;
p = ipipe_this_cpu_context(ipd);
if (__test_and_clear_bit(irq, p->irqpend_map)) {
__set_bit(irq, p->irqheld_map);
l2b = irq / BITS_PER_LONG;
if (p->irqpend_map[l2b] == 0) {
__clear_bit(l2b, p->irqpend_2map);
l1b = l2b / BITS_PER_LONG;
if (p->irqpend_2map[l1b] == 0) {
__clear_bit(l1b, p->irqpend_1map);
l0b = l1b / BITS_PER_LONG;
if (p->irqpend_1map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
}
}
}
}
EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
/* Must be called hw IRQs off. */
void __ipipe_unlock_irq(unsigned int irq)
{
struct ipipe_domain *ipd = ipipe_root_domain;
struct ipipe_percpu_domain_data *p;
int l0b, l1b, l2b, cpu;
IPIPE_WARN_ONCE(!hard_irqs_disabled());
if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
return;
l0b = irq / (BITS_PER_LONG * BITS_PER_LONG * BITS_PER_LONG);
l1b = irq / (BITS_PER_LONG * BITS_PER_LONG);
l2b = irq / BITS_PER_LONG;
for_each_online_cpu(cpu) {
p = ipipe_this_cpu_root_context();
if (test_and_clear_bit(irq, p->irqheld_map)) {
/* We need atomic ops here: */
set_bit(irq, p->irqpend_map);
set_bit(l2b, p->irqpend_2map);
set_bit(l1b, p->irqpend_1map);
set_bit(l0b, &p->irqpend_0map);
}
}
}
EXPORT_SYMBOL_GPL(__ipipe_unlock_irq);
#define wmul1(__n) ((__n) * BITS_PER_LONG)
#define wmul2(__n) (wmul1(__n) * BITS_PER_LONG)
#define wmul3(__n) (wmul2(__n) * BITS_PER_LONG)
static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
{
unsigned long l0m, l1m, l2m, l3m;
int l0b, l1b, l2b, l3b;
unsigned int irq;
l0m = p->irqpend_0map;
if (unlikely(l0m == 0))
return -1;
l0b = __ipipe_ffnz(l0m);
irq = wmul3(l0b);
l1m = p->irqpend_1map[l0b];
if (unlikely(l1m == 0))
return -1;
l1b = __ipipe_ffnz(l1m);
irq += wmul2(l1b);
l2m = p->irqpend_2map[wmul1(l0b) + l1b];
if (unlikely(l2m == 0))
return -1;
l2b = __ipipe_ffnz(l2m);
irq += wmul1(l2b);
l3m = p->irqpend_map[wmul2(l0b) + wmul1(l1b) + l2b];
if (unlikely(l3m == 0))
return -1;
l3b = __ipipe_ffnz(l3m);
irq += l3b;
__clear_bit(irq, p->irqpend_map);
if (p->irqpend_map[irq / BITS_PER_LONG] == 0) {
__clear_bit(l2b, &p->irqpend_2map[wmul1(l0b) + l1b]);
if (p->irqpend_2map[wmul1(l0b) + l1b] == 0) {
__clear_bit(l1b, &p->irqpend_1map[l0b]);
if (p->irqpend_1map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
}
}
return irq;
}
#elif __IPIPE_IRQMAP_LEVELS == 3
/* Must be called hw IRQs off. */
void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
{
......@@ -627,9 +766,9 @@ void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
l1b = irq / BITS_PER_LONG;
if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
__set_bit(irq, p->irqpend_lomap);
__set_bit(l1b, p->irqpend_mdmap);
__set_bit(l0b, &p->irqpend_himap);
__set_bit(irq, p->irqpend_map);
__set_bit(l1b, p->irqpend_1map);
__set_bit(l0b, &p->irqpend_0map);
} else
__set_bit(irq, p->irqheld_map);
......@@ -660,12 +799,12 @@ void __ipipe_lock_irq(unsigned int irq)
l1b = irq / BITS_PER_LONG;
p = ipipe_this_cpu_context(ipd);
if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
if (__test_and_clear_bit(irq, p->irqpend_map)) {
__set_bit(irq, p->irqheld_map);
if (p->irqpend_lomap[l1b] == 0) {
__clear_bit(l1b, p->irqpend_mdmap);
if (p->irqpend_mdmap[l0b] == 0)
__clear_bit(l0b, &p->irqpend_himap);
if (p->irqpend_map[l1b] == 0) {
__clear_bit(l1b, p->irqpend_1map);
if (p->irqpend_1map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
}
}
}
......@@ -690,9 +829,9 @@ void __ipipe_unlock_irq(unsigned int irq)
p = ipipe_this_cpu_root_context();
if (test_and_clear_bit(irq, p->irqheld_map)) {
/* We need atomic ops here: */
set_bit(irq, p->irqpend_lomap);
set_bit(l1b, p->irqpend_mdmap);
set_bit(l0b, &p->irqpend_himap);
set_bit(irq, p->irqpend_map);
set_bit(l1b, p->irqpend_1map);
set_bit(l0b, &p->irqpend_0map);
}
}
}
......@@ -704,42 +843,34 @@ static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
unsigned long l0m, l1m, l2m;
unsigned int irq;
l0m = p->irqpend_himap;
l0m = p->irqpend_0map;
if (unlikely(l0m == 0))
return -1;
l0b = __ipipe_ffnz(l0m);
l1m = p->irqpend_mdmap[l0b];
l1m = p->irqpend_1map[l0b];
if (unlikely(l1m == 0))
return -1;
l1b = __ipipe_ffnz(l1m) + l0b * BITS_PER_LONG;
l2m = p->irqpend_lomap[l1b];
l2m = p->irqpend_map[l1b];
if (unlikely(l2m == 0))
return -1;
l2b = __ipipe_ffnz(l2m);
irq = l1b * BITS_PER_LONG + l2b;
__clear_bit(irq, p->irqpend_lomap);
if (p->irqpend_lomap[l1b] == 0) {
__clear_bit(l1b, p->irqpend_mdmap);
if (p->irqpend_mdmap[l0b] == 0)
__clear_bit(l0b, &p->irqpend_himap);
__clear_bit(irq, p->irqpend_map);
if (p->irqpend_map[l1b] == 0) {
__clear_bit(l1b, p->irqpend_1map);
if (p->irqpend_1map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
}
return irq;
}
#else /* __IPIPE_2LEVEL_IRQMAP */
/* Must be called hw IRQs off. */
static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p,
unsigned int irq)
{
__set_bit(irq, p->irqheld_map);
p->irqall[irq]++;
}
#else /* __IPIPE_IRQMAP_LEVELS == 2 */
/* Must be called hw IRQs off. */
void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
......@@ -750,8 +881,8 @@ void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq)
IPIPE_WARN_ONCE(!hard_irqs_disabled());
if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
__set_bit(irq, p->irqpend_lomap);
__set_bit(l0b, &p->irqpend_himap);
__set_bit(irq, p->irqpend_map);
__set_bit(l0b, &p->irqpend_0map);
} else
__set_bit(irq, p->irqheld_map);
......@@ -772,10 +903,10 @@ void __ipipe_lock_irq(unsigned int irq)
return;
p = ipipe_this_cpu_root_context();
if (__test_and_clear_bit(irq, p->irqpend_lomap)) {
if (__test_and_clear_bit(irq, p->irqpend_map)) {
__set_bit(irq, p->irqheld_map);
if (p->irqpend_lomap[l0b] == 0)
__clear_bit(l0b, &p->irqpend_himap);
if (p->irqpend_map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
}
}
EXPORT_SYMBOL_GPL(__ipipe_lock_irq);
......@@ -796,8 +927,8 @@ void __ipipe_unlock_irq(unsigned int irq)
p = ipipe_percpu_context(ipd, cpu);
if (test_and_clear_bit(irq, p->irqheld_map)) {
/* We need atomic ops here: */
set_bit(irq, p->irqpend_lomap);
set_bit(l0b, &p->irqpend_himap);
set_bit(irq, p->irqpend_map);
set_bit(l0b, &p->irqpend_0map);
}
}
}
......@@ -808,24 +939,24 @@ static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p)
unsigned long l0m, l1m;
int l0b, l1b;
l0m = p->irqpend_himap;
l0m = p->irqpend_0map;
if (unlikely(l0m == 0))
return -1;
l0b = __ipipe_ffnz(l0m);
l1m = p->irqpend_lomap[l0b];
l1m = p->irqpend_map[l0b];
if (unlikely(l1m == 0))
return -1;
l1b = __ipipe_ffnz(l1m);
__clear_bit(l1b, &p->irqpend_lomap[l0b]);
if (p->irqpend_lomap[l0b] == 0)
__clear_bit(l0b, &p->irqpend_himap);
__clear_bit(l1b, &p->irqpend_map[l0b]);
if (p->irqpend_map[l0b] == 0)
__clear_bit(l0b, &p->irqpend_0map);
return l0b * BITS_PER_LONG + l1b;
}
#endif /* __IPIPE_2LEVEL_IRQMAP */
#endif
void __ipipe_do_sync_pipeline(struct ipipe_domain *top)
{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment