smp.c 25.4 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  arch/s390/kernel/smp.c
 *
4
 *    Copyright IBM Corp. 1999, 2009
Linus Torvalds's avatar
Linus Torvalds committed
5
 *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6
7
 *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
 *		 Heiko Carstens (heiko.carstens@de.ibm.com)
Linus Torvalds's avatar
Linus Torvalds committed
8
 *
9
 *  based on other smp stuff by
Linus Torvalds's avatar
Linus Torvalds committed
10
11
12
13
14
15
16
17
18
19
20
21
22
 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *    (c) 1998 Ingo Molnar
 *
 * We work with logical cpu numbering everywhere we can. The only
 * functions using the real cpu address (got from STAP) are the sigp
 * functions. For all other functions we use the identity mapping.
 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
 * used e.g. to find the idle task belonging to a logical cpu. Every array
 * in the kernel is sorted by the logical cpu number and not by the physical
 * one which is causing all the confusion with __cpu_logical_map and
 * cpu_number_map in other architectures.
 */

23
24
25
#define KMSG_COMPONENT "cpu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
26
27
28
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
29
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
31
32
33
34
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
35
#include <linux/irqflags.h>
Linus Torvalds's avatar
Linus Torvalds committed
36
#include <linux/cpu.h>
37
#include <linux/timex.h>
Michael Holzheu's avatar
Michael Holzheu committed
38
#include <linux/bootmem.h>
Michael Holzheu's avatar
Michael Holzheu committed
39
#include <asm/ipl.h>
40
#include <asm/setup.h>
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
#include <asm/sigp.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
#include <asm/s390_ext.h>
#include <asm/cpcmd.h>
#include <asm/tlbflush.h>
47
#include <asm/timer.h>
Michael Holzheu's avatar
Michael Holzheu committed
48
#include <asm/lowcore.h>
49
#include <asm/sclp.h>
50
#include <asm/cputime.h>
51
#include <asm/vdso.h>
52
#include <asm/cpu.h>
53
#include "entry.h"
Linus Torvalds's avatar
Linus Torvalds committed
54
55
56

static struct task_struct *current_set[NR_CPUS];

57
58
59
60
61
62
63
64
static u8 smp_cpu_type;
static int smp_use_sigp_detection;

enum s390_cpu_state {
	CPU_STATE_STANDBY,
	CPU_STATE_CONFIGURED,
};

65
DEFINE_MUTEX(smp_cpu_state_mutex);
66
int smp_cpu_polarization[NR_CPUS];
67
static int smp_cpu_state[NR_CPUS];
68
static int cpu_management;
69
70
71

static DEFINE_PER_CPU(struct cpu, cpu_devices);

Linus Torvalds's avatar
Linus Torvalds committed
72
73
static void smp_ext_bitcall(int, ec_bit_sig);

74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
static int cpu_stopped(int cpu)
{
	__u32 status;

	switch (signal_processor_ps(&status, 0, cpu, sigp_sense)) {
	case sigp_status_stored:
		/* Check for stopped and check stop state */
		if (status & 0x50)
			return 1;
		break;
	default:
		break;
	}
	return 0;
}

90
void smp_send_stop(void)
Linus Torvalds's avatar
Linus Torvalds committed
91
{
92
	int cpu, rc;
Linus Torvalds's avatar
Linus Torvalds committed
93

94
95
	/* Disable all interrupts/machine checks */
	__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
96
	trace_hardirqs_off();
Linus Torvalds's avatar
Linus Torvalds committed
97

98
	/* stop all processors */
Linus Torvalds's avatar
Linus Torvalds committed
99
100
101
102
	for_each_online_cpu(cpu) {
		if (cpu == smp_processor_id())
			continue;
		do {
103
			rc = signal_processor(cpu, sigp_stop);
104
		} while (rc == sigp_busy);
Linus Torvalds's avatar
Linus Torvalds committed
105

106
		while (!cpu_stopped(cpu))
Heiko Carstens's avatar
Heiko Carstens committed
107
108
109
110
			cpu_relax();
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
111
112
113
114
115
/*
 * This is the main routine where commands issued by other
 * cpus are handled.
 */

116
static void do_ext_call_interrupt(__u16 code)
Linus Torvalds's avatar
Linus Torvalds committed
117
{
118
	unsigned long bits;
Linus Torvalds's avatar
Linus Torvalds committed
119

120
121
122
123
124
125
	/*
	 * handle bit signal external calls
	 *
	 * For the ec_schedule signal we have to do nothing. All the work
	 * is done automatically when we return from the interrupt.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
126
127
	bits = xchg(&S390_lowcore.ext_call_fast, 0);

128
	if (test_bit(ec_call_function, &bits))
129
130
131
132
		generic_smp_call_function_interrupt();

	if (test_bit(ec_call_function_single, &bits))
		generic_smp_call_function_single_interrupt();
Linus Torvalds's avatar
Linus Torvalds committed
133
134
135
136
137
138
139
140
}

/*
 * Send an external call sigp to another cpu and return without waiting
 * for its completion.
 */
static void smp_ext_bitcall(int cpu, ec_bit_sig sig)
{
141
142
143
	/*
	 * Set signaling bit in lowcore of target cpu and kick it
	 */
Linus Torvalds's avatar
Linus Torvalds committed
144
	set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
145
	while (signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds's avatar
Linus Torvalds committed
146
147
148
		udelay(10);
}

149
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
150
151
152
{
	int cpu;

153
	for_each_cpu(cpu, mask)
154
155
156
157
158
159
160
161
		smp_ext_bitcall(cpu, ec_call_function);
}

void arch_send_call_function_single_ipi(int cpu)
{
	smp_ext_bitcall(cpu, ec_call_function_single);
}

162
#ifndef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
163
164
165
/*
 * this function sends a 'purge tlb' signal to another CPU.
 */
166
static void smp_ptlb_callback(void *info)
Linus Torvalds's avatar
Linus Torvalds committed
167
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
168
	__tlb_flush_local();
Linus Torvalds's avatar
Linus Torvalds committed
169
170
171
172
}

void smp_ptlb_all(void)
{
173
	on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds's avatar
Linus Torvalds committed
174
175
}
EXPORT_SYMBOL(smp_ptlb_all);
176
#endif /* ! CONFIG_64BIT */
Linus Torvalds's avatar
Linus Torvalds committed
177
178
179
180
181
182
183
184

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
void smp_send_reschedule(int cpu)
{
185
	smp_ext_bitcall(cpu, ec_schedule);
Linus Torvalds's avatar
Linus Torvalds committed
186
187
188
189
190
}

/*
 * parameter area for the set/clear control bit callbacks
 */
191
struct ec_creg_mask_parms {
Linus Torvalds's avatar
Linus Torvalds committed
192
193
	unsigned long orvals[16];
	unsigned long andvals[16];
194
};
Linus Torvalds's avatar
Linus Torvalds committed
195
196
197
198

/*
 * callback for setting/clearing control bits
 */
199
200
static void smp_ctl_bit_callback(void *info)
{
201
	struct ec_creg_mask_parms *pp = info;
Linus Torvalds's avatar
Linus Torvalds committed
202
203
	unsigned long cregs[16];
	int i;
204

205
206
	__ctl_store(cregs, 0, 15);
	for (i = 0; i <= 15; i++)
Linus Torvalds's avatar
Linus Torvalds committed
207
		cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
208
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
209
210
211
212
213
}

/*
 * Set a bit in a control register of all cpus
 */
214
215
216
void smp_ctl_set_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
217

218
219
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
220
	parms.orvals[cr] = 1 << bit;
221
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
222
}
223
EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds's avatar
Linus Torvalds committed
224
225
226
227

/*
 * Clear a bit in a control register of all cpus
 */
228
229
230
void smp_ctl_clear_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
231

232
233
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
234
	parms.andvals[cr] = ~(1L << bit);
235
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
236
}
237
EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds's avatar
Linus Torvalds committed
238

239
240
241
242
243
244
245
/*
 * In early ipl state a temp. logically cpu number is needed, so the sigp
 * functions can be used to sense other cpus. Since NR_CPUS is >= 2 on
 * CONFIG_SMP and the ipl cpu is logical cpu 0, it must be 1.
 */
#define CPU_INIT_NO	1

246
#ifdef CONFIG_ZFCPDUMP
Michael Holzheu's avatar
Michael Holzheu committed
247
248
249
250
251
252
253
254
255
256

/*
 * zfcpdump_prefix_array holds prefix registers for the following scenario:
 * 64 bit zfcpdump kernel and 31 bit kernel which is to be dumped. We have to
 * save its prefix registers, since they get lost, when switching from 31 bit
 * to 64 bit.
 */
unsigned int zfcpdump_prefix_array[NR_CPUS + 1] \
	__attribute__((__section__(".data")));

257
static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
Michael Holzheu's avatar
Michael Holzheu committed
258
259
260
{
	if (ipl_info.type != IPL_TYPE_FCP_DUMP)
		return;
261
	if (cpu >= NR_CPUS) {
262
263
		pr_warning("CPU %i exceeds the maximum %i and is excluded from "
			   "the dump\n", cpu, NR_CPUS - 1);
264
		return;
Michael Holzheu's avatar
Michael Holzheu committed
265
	}
266
	zfcpdump_save_areas[cpu] = kmalloc(sizeof(union save_area), GFP_KERNEL);
267
268
269
	__cpu_logical_map[CPU_INIT_NO] = (__u16) phy_cpu;
	while (signal_processor(CPU_INIT_NO, sigp_stop_and_store_status) ==
	       sigp_busy)
270
271
272
273
274
275
276
277
		cpu_relax();
	memcpy(zfcpdump_save_areas[cpu],
	       (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
	       SAVE_AREA_SIZE);
#ifdef CONFIG_64BIT
	/* copy original prefix register */
	zfcpdump_save_areas[cpu]->s390x.pref_reg = zfcpdump_prefix_array[cpu];
#endif
Michael Holzheu's avatar
Michael Holzheu committed
278
279
280
281
282
283
}

union save_area *zfcpdump_save_areas[NR_CPUS + 1];
EXPORT_SYMBOL_GPL(zfcpdump_save_areas);

#else
284
285
286

static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }

287
#endif /* CONFIG_ZFCPDUMP */
Michael Holzheu's avatar
Michael Holzheu committed
288

289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
static int cpu_known(int cpu_id)
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (__cpu_logical_map[cpu] == cpu_id)
			return 1;
	}
	return 0;
}

static int smp_rescan_cpus_sigp(cpumask_t avail)
{
	int cpu_id, logical_cpu;

304
305
	logical_cpu = cpumask_first(&avail);
	if (logical_cpu >= nr_cpu_ids)
306
		return 0;
307
	for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
308
309
310
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
311
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
312
313
314
315
		if (!cpu_stopped(logical_cpu))
			continue;
		cpu_set(logical_cpu, cpu_present_map);
		smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
316
317
		logical_cpu = cpumask_next(logical_cpu, &avail);
		if (logical_cpu >= nr_cpu_ids)
318
319
320
321
322
			break;
	}
	return 0;
}

323
static int smp_rescan_cpus_sclp(cpumask_t avail)
324
325
326
327
328
{
	struct sclp_cpu_info *info;
	int cpu_id, logical_cpu, cpu;
	int rc;

329
330
	logical_cpu = cpumask_first(&avail);
	if (logical_cpu >= nr_cpu_ids)
331
		return 0;
332
	info = kmalloc(sizeof(*info), GFP_KERNEL);
333
334
335
336
337
338
339
340
341
342
343
344
	if (!info)
		return -ENOMEM;
	rc = sclp_get_cpu_info(info);
	if (rc)
		goto out;
	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_id = info->cpu[cpu].address;
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
345
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
346
347
348
349
350
		cpu_set(logical_cpu, cpu_present_map);
		if (cpu >= info->configured)
			smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
		else
			smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
351
352
		logical_cpu = cpumask_next(logical_cpu, &avail);
		if (logical_cpu >= nr_cpu_ids)
353
354
355
			break;
	}
out:
356
	kfree(info);
357
358
359
	return rc;
}

360
static int __smp_rescan_cpus(void)
361
362
363
{
	cpumask_t avail;

364
	cpus_xor(avail, cpu_possible_map, cpu_present_map);
365
366
367
368
	if (smp_use_sigp_detection)
		return smp_rescan_cpus_sigp(avail);
	else
		return smp_rescan_cpus_sclp(avail);
Linus Torvalds's avatar
Linus Torvalds committed
369
370
}

371
372
373
374
375
376
377
378
static void __init smp_detect_cpus(void)
{
	unsigned int cpu, c_cpus, s_cpus;
	struct sclp_cpu_info *info;
	u16 boot_cpu_addr, cpu_addr;

	c_cpus = 1;
	s_cpus = 0;
379
	boot_cpu_addr = __cpu_logical_map[0];
380
381
382
383
384
385
	info = kmalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		panic("smp_detect_cpus failed to allocate memory\n");
	/* Use sigp detection algorithm if sclp doesn't work. */
	if (sclp_get_cpu_info(info)) {
		smp_use_sigp_detection = 1;
386
		for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
			if (cpu == boot_cpu_addr)
				continue;
			__cpu_logical_map[CPU_INIT_NO] = cpu;
			if (!cpu_stopped(CPU_INIT_NO))
				continue;
			smp_get_save_area(c_cpus, cpu);
			c_cpus++;
		}
		goto out;
	}

	if (info->has_cpu_type) {
		for (cpu = 0; cpu < info->combined; cpu++) {
			if (info->cpu[cpu].address == boot_cpu_addr) {
				smp_cpu_type = info->cpu[cpu].type;
				break;
			}
		}
	}

	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_addr = info->cpu[cpu].address;
		if (cpu_addr == boot_cpu_addr)
			continue;
		__cpu_logical_map[CPU_INIT_NO] = cpu_addr;
		if (!cpu_stopped(CPU_INIT_NO)) {
			s_cpus++;
			continue;
		}
		smp_get_save_area(c_cpus, cpu_addr);
		c_cpus++;
	}
out:
	kfree(info);
423
	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
424
	get_online_cpus();
425
	__smp_rescan_cpus();
426
	put_online_cpus();
427
428
}

Linus Torvalds's avatar
Linus Torvalds committed
429
/*
430
 *	Activate a secondary processor.
Linus Torvalds's avatar
Linus Torvalds committed
431
 */
432
int __cpuinit start_secondary(void *cpuvoid)
Linus Torvalds's avatar
Linus Torvalds committed
433
{
434
435
	/* Setup the cpu */
	cpu_init();
436
	preempt_disable();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
437
	/* Enable TOD clock interrupts on the secondary cpu. */
438
	init_cpu_timer();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
439
	/* Enable cpu timer interrupts on the secondary cpu. */
440
	init_cpu_vtimer();
Linus Torvalds's avatar
Linus Torvalds committed
441
	/* Enable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
442
443
	pfault_init();

444
445
	/* call cpu notifiers */
	notify_cpu_starting(smp_processor_id());
Linus Torvalds's avatar
Linus Torvalds committed
446
	/* Mark this cpu as online */
447
	ipi_call_lock();
Linus Torvalds's avatar
Linus Torvalds committed
448
	cpu_set(smp_processor_id(), cpu_online_map);
449
	ipi_call_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
450
451
	/* Switch on interrupts */
	local_irq_enable();
452
	/* Print info about this processor */
453
	print_cpu_info();
454
455
456
	/* cpu_idle will call schedule for us */
	cpu_idle();
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
}

static void __init smp_create_idle(unsigned int cpu)
{
	struct task_struct *p;

	/*
	 *  don't care about the psw and regs settings since we'll never
	 *  reschedule the forked task.
	 */
	p = fork_idle(cpu);
	if (IS_ERR(p))
		panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
	current_set[cpu] = p;
}

473
474
475
476
477
static int __cpuinit smp_alloc_lowcore(int cpu)
{
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;

478
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
479
480
481
482
	if (!lowcore)
		return -ENOMEM;
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
	panic_stack = __get_free_page(GFP_KERNEL);
483
484
	if (!panic_stack || !async_stack)
		goto out;
485
486
	memcpy(lowcore, &S390_lowcore, 512);
	memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
487
488
489
490
491
492
493
494
495
	lowcore->async_stack = async_stack + ASYNC_SIZE;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;

#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		unsigned long save_area;

		save_area = get_zeroed_page(GFP_KERNEL);
		if (!save_area)
496
			goto out;
497
498
		lowcore->extended_save_area_addr = (u32) save_area;
	}
499
500
501
#else
	if (vdso_alloc_per_cpu(cpu, lowcore))
		goto out;
502
503
504
505
#endif
	lowcore_ptr[cpu] = lowcore;
	return 0;

506
out:
507
	free_page(panic_stack);
508
	free_pages(async_stack, ASYNC_ORDER);
509
	free_pages((unsigned long) lowcore, LC_ORDER);
510
511
512
513
514
515
516
517
518
519
520
	return -ENOMEM;
}

static void smp_free_lowcore(int cpu)
{
	struct _lowcore *lowcore;

	lowcore = lowcore_ptr[cpu];
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		free_page((unsigned long) lowcore->extended_save_area_addr);
521
522
#else
	vdso_free_per_cpu(cpu, lowcore);
523
524
525
#endif
	free_page(lowcore->panic_stack - PAGE_SIZE);
	free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
526
	free_pages((unsigned long) lowcore, LC_ORDER);
527
528
529
	lowcore_ptr[cpu] = NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
530
/* Upping and downing of CPUs */
531
int __cpuinit __cpu_up(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
532
533
{
	struct task_struct *idle;
534
	struct _lowcore *cpu_lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
535
	struct stack_frame *sf;
536
	sigp_ccode ccode;
537
	u32 lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
538

539
540
	if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
		return -EIO;
541
542
	if (smp_alloc_lowcore(cpu))
		return -ENOMEM;
543
544
545
546
547
548
549
550
551
552
553
	do {
		ccode = signal_processor(cpu, sigp_initial_cpu_reset);
		if (ccode == sigp_busy)
			udelay(10);
		if (ccode == sigp_not_operational)
			goto err_out;
	} while (ccode == sigp_busy);

	lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
	while (signal_processor_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
554
555

	idle = current_set[cpu];
556
	cpu_lowcore = lowcore_ptr[cpu];
Linus Torvalds's avatar
Linus Torvalds committed
557
	cpu_lowcore->kernel_stack = (unsigned long)
558
		task_stack_page(idle) + THREAD_SIZE;
559
	cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
Linus Torvalds's avatar
Linus Torvalds committed
560
561
562
563
564
565
	sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
				     - sizeof(struct pt_regs)
				     - sizeof(struct stack_frame));
	memset(sf, 0, sizeof(struct stack_frame));
	sf->gprs[9] = (unsigned long) sf;
	cpu_lowcore->save_area[15] = (unsigned long) sf;
566
	__ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
567
568
569
	asm volatile(
		"	stam	0,15,0(%0)"
		: : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
Linus Torvalds's avatar
Linus Torvalds committed
570
	cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
571
	cpu_lowcore->current_task = (unsigned long) idle;
572
	cpu_lowcore->cpu_nr = cpu;
573
	cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
574
	cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
575
	cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
Linus Torvalds's avatar
Linus Torvalds committed
576
	eieio();
Michael Ryan's avatar
Michael Ryan committed
577

578
	while (signal_processor(cpu, sigp_restart) == sigp_busy)
Michael Ryan's avatar
Michael Ryan committed
579
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
580
581
582
583

	while (!cpu_online(cpu))
		cpu_relax();
	return 0;
584
585
586
587

err_out:
	smp_free_lowcore(cpu);
	return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
588
589
}

590
static int __init setup_possible_cpus(char *s)
591
{
592
	int pcpus, cpu;
593

594
	pcpus = simple_strtoul(s, NULL, 0);
595
596
	init_cpu_possible(cpumask_of(0));
	for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
597
		set_cpu_possible(cpu, true);
598
599
600
601
	return 0;
}
early_param("possible_cpus", setup_possible_cpus);

602
603
#ifdef CONFIG_HOTPLUG_CPU

604
int __cpu_disable(void)
Linus Torvalds's avatar
Linus Torvalds committed
605
{
606
	struct ec_creg_mask_parms cr_parms;
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
607
	int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
608

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
609
	cpu_clear(cpu, cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
610
611

	/* Disable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
612
	pfault_fini();
Linus Torvalds's avatar
Linus Torvalds committed
613

614
615
	memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
	memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
616

617
	/* disable all external interrupts */
Linus Torvalds's avatar
Linus Torvalds committed
618
	cr_parms.orvals[0] = 0;
619
620
	cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
				1 << 11 | 1 << 10 | 1 <<  6 | 1 <<  4);
Linus Torvalds's avatar
Linus Torvalds committed
621
622
	/* disable all I/O interrupts */
	cr_parms.orvals[6] = 0;
623
624
	cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
				1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
Linus Torvalds's avatar
Linus Torvalds committed
625
626
	/* disable most machine checks */
	cr_parms.orvals[14] = 0;
627
628
	cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
				 1 << 25 | 1 << 24);
629

Linus Torvalds's avatar
Linus Torvalds committed
630
631
632
633
634
	smp_ctl_bit_callback(&cr_parms);

	return 0;
}

635
void __cpu_die(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
636
637
{
	/* Wait until target cpu is down */
638
	while (!cpu_stopped(cpu))
Linus Torvalds's avatar
Linus Torvalds committed
639
		cpu_relax();
640
641
	while (signal_processor_p(0, cpu, sigp_set_prefix) == sigp_busy)
		udelay(10);
642
	smp_free_lowcore(cpu);
643
	pr_info("Processor %d stopped\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
644
645
}

646
void cpu_die(void)
Linus Torvalds's avatar
Linus Torvalds committed
647
648
{
	idle_task_exit();
649
650
	while (signal_processor(smp_processor_id(), sigp_stop) == sigp_busy)
		cpu_relax();
651
	for (;;);
Linus Torvalds's avatar
Linus Torvalds committed
652
653
}

654
655
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
656
657
void __init smp_prepare_cpus(unsigned int max_cpus)
{
658
659
660
661
662
#ifndef CONFIG_64BIT
	unsigned long save_area = 0;
#endif
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
663
	unsigned int cpu;
664

665
666
	smp_detect_cpus();

667
668
669
	/* request the 0x1201 emergency signal external interrupt */
	if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1201");
670
	print_cpu_info();
Linus Torvalds's avatar
Linus Torvalds committed
671

672
	/* Reallocate current lowcore, but keep its contents. */
673
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
674
675
	panic_stack = __get_free_page(GFP_KERNEL);
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
676
	BUG_ON(!lowcore || !panic_stack || !async_stack);
677
#ifndef CONFIG_64BIT
678
	if (MACHINE_HAS_IEEE)
679
		save_area = get_zeroed_page(GFP_KERNEL);
680
#endif
681
682
683
684
685
686
687
688
689
690
691
692
693
	local_irq_disable();
	local_mcck_disable();
	lowcore_ptr[smp_processor_id()] = lowcore;
	*lowcore = S390_lowcore;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;
	lowcore->async_stack = async_stack + ASYNC_SIZE;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		lowcore->extended_save_area_addr = (u32) save_area;
#endif
	set_prefix((u32)(unsigned long) lowcore);
	local_mcck_enable();
	local_irq_enable();
694
695
696
697
#ifdef CONFIG_64BIT
	if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
		BUG();
#endif
698
	for_each_possible_cpu(cpu)
Linus Torvalds's avatar
Linus Torvalds committed
699
700
701
702
		if (cpu != smp_processor_id())
			smp_create_idle(cpu);
}

703
void __init smp_prepare_boot_cpu(void)
Linus Torvalds's avatar
Linus Torvalds committed
704
705
706
{
	BUG_ON(smp_processor_id() != 0);

707
708
	current_thread_info()->cpu = 0;
	cpu_set(0, cpu_present_map);
Linus Torvalds's avatar
Linus Torvalds committed
709
710
711
	cpu_set(0, cpu_online_map);
	S390_lowcore.percpu_offset = __per_cpu_offset[0];
	current_set[0] = current;
712
	smp_cpu_state[0] = CPU_STATE_CONFIGURED;
713
	smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
Linus Torvalds's avatar
Linus Torvalds committed
714
715
}

716
void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds's avatar
Linus Torvalds committed
717
718
719
720
721
722
723
724
725
726
727
{
}

/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
728
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
729
730
}

731
#ifdef CONFIG_HOTPLUG_CPU
732
733
static ssize_t cpu_configure_show(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
734
735
736
737
738
739
740
741
742
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

743
744
745
static ssize_t cpu_configure_store(struct sys_device *dev,
				  struct sysdev_attribute *attr,
				  const char *buf, size_t count)
746
747
748
749
750
751
752
753
754
755
{
	int cpu = dev->id;
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;

756
	get_online_cpus();
757
	mutex_lock(&smp_cpu_state_mutex);
758
759
760
761
762
763
764
765
	rc = -EBUSY;
	if (cpu_online(cpu))
		goto out;
	rc = 0;
	switch (val) {
	case 0:
		if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
			rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
766
			if (!rc) {
767
				smp_cpu_state[cpu] = CPU_STATE_STANDBY;
768
769
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
770
771
772
773
774
		}
		break;
	case 1:
		if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
			rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
775
			if (!rc) {
776
				smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
777
778
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
779
780
781
782
783
784
785
		}
		break;
	default:
		break;
	}
out:
	mutex_unlock(&smp_cpu_state_mutex);
786
	put_online_cpus();
787
788
789
790
791
	return rc ? rc : count;
}
static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
#endif /* CONFIG_HOTPLUG_CPU */

792
793
static ssize_t cpu_polarization_show(struct sys_device *dev,
				     struct sysdev_attribute *attr, char *buf)
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	switch (smp_cpu_polarization[cpu]) {
	case POLARIZATION_HRZ:
		count = sprintf(buf, "horizontal\n");
		break;
	case POLARIZATION_VL:
		count = sprintf(buf, "vertical:low\n");
		break;
	case POLARIZATION_VM:
		count = sprintf(buf, "vertical:medium\n");
		break;
	case POLARIZATION_VH:
		count = sprintf(buf, "vertical:high\n");
		break;
	default:
		count = sprintf(buf, "unknown\n");
		break;
	}
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);

821
822
static ssize_t show_cpu_address(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
823
824
825
826
827
828
829
830
831
832
833
{
	return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
}
static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);


static struct attribute *cpu_common_attrs[] = {
#ifdef CONFIG_HOTPLUG_CPU
	&attr_configure.attr,
#endif
	&attr_address.attr,
834
	&attr_polarization.attr,
835
836
837
838
839
840
	NULL,
};

static struct attribute_group cpu_common_attr_group = {
	.attrs = cpu_common_attrs,
};
Linus Torvalds's avatar
Linus Torvalds committed
841

842
843
static ssize_t show_capability(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
844
845
846
847
848
849
850
851
852
853
854
{
	unsigned int capability;
	int rc;

	rc = get_cpu_capability(&capability);
	if (rc)
		return rc;
	return sprintf(buf, "%u\n", capability);
}
static SYSDEV_ATTR(capability, 0444, show_capability, NULL);

855
856
static ssize_t show_idle_count(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
857
858
859
{
	struct s390_idle_data *idle;
	unsigned long long idle_count;
860
	unsigned int sequence;
861
862

	idle = &per_cpu(s390_idle, dev->id);
863
864
865
866
867
repeat:
	sequence = idle->sequence;
	smp_rmb();
	if (sequence & 1)
		goto repeat;
868
	idle_count = idle->idle_count;
869
870
	if (idle->idle_enter)
		idle_count++;
871
872
873
	smp_rmb();
	if (idle->sequence != sequence)
		goto repeat;
874
875
876
877
	return sprintf(buf, "%llu\n", idle_count);
}
static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);

878
879
static ssize_t show_idle_time(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
880
881
{
	struct s390_idle_data *idle;
882
	unsigned long long now, idle_time, idle_enter;
883
	unsigned int sequence;
884
885

	idle = &per_cpu(s390_idle, dev->id);
886
	now = get_clock();
887
888
889
890
891
repeat:
	sequence = idle->sequence;
	smp_rmb();
	if (sequence & 1)
		goto repeat;
892
893
894
895
	idle_time = idle->idle_time;
	idle_enter = idle->idle_enter;
	if (idle_enter != 0ULL && idle_enter < now)
		idle_time += now - idle_enter;
896
897
898
	smp_rmb();
	if (idle->sequence != sequence)
		goto repeat;
899
	return sprintf(buf, "%llu\n", idle_time >> 12);
900
}
901
static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
902

903
static struct attribute *cpu_online_attrs[] = {
904
905
	&attr_capability.attr,
	&attr_idle_count.attr,
906
	&attr_idle_time_us.attr,
907
908
909
	NULL,
};

910
911
static struct attribute_group cpu_online_attr_group = {
	.attrs = cpu_online_attrs,
912
913
};

914
915
916
917
918
919
static int __cpuinit smp_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned int)(long)hcpu;
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
920
	struct s390_idle_data *idle;
921
922
923

	switch (action) {
	case CPU_ONLINE:
924
	case CPU_ONLINE_FROZEN:
925
		idle = &per_cpu(s390_idle, cpu);
926
		memset(idle, 0, sizeof(struct s390_idle_data));
927
		if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
928
929
930
			return NOTIFY_BAD;
		break;
	case CPU_DEAD:
931
	case CPU_DEAD_FROZEN:
932
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
933
934
935
936
937
938
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata smp_cpu_nb = {
939
	.notifier_call = smp_cpu_notify,
940
941
};

942
static int __devinit smp_add_present_cpu(int cpu)
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
{
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
	int rc;

	c->hotpluggable = 1;
	rc = register_cpu(c, cpu);
	if (rc)
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
	if (rc)
		goto out_cpu;
	if (!cpu_online(cpu))
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
	if (!rc)
		return 0;
	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
out_cpu:
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu(c);
#endif
out:
	return rc;
}

#ifdef CONFIG_HOTPLUG_CPU
970

971
int __ref smp_rescan_cpus(void)
972
973
974
975
976
{
	cpumask_t newcpus;
	int cpu;
	int rc;

977
	get_online_cpus();
978
	mutex_lock(&smp_cpu_state_mutex);
979
	newcpus = cpu_present_map;
980
	rc = __smp_rescan_cpus();
981
982
983
984
985
986
987
988
989
990
991
	if (rc)
		goto out;
	cpus_andnot(newcpus, cpu_present_map, newcpus);
	for_each_cpu_mask(cpu, newcpus) {
		rc = smp_add_present_cpu(cpu);
		if (rc)
			cpu_clear(cpu, cpu_present_map);
	}
	rc = 0;
out:
	mutex_unlock(&smp_cpu_state_mutex);
992
	put_online_cpus();
993
994
	if (!cpus_empty(newcpus))
		topology_schedule_update();
995
996
997
	return rc;
}

998
static ssize_t __ref rescan_store(struct sysdev_class *class, const char *buf,
999
1000
1001
1002
1003
				  size_t count)
{
	int rc;

	rc = smp_rescan_cpus();
1004
1005
	return rc ? rc : count;
}
1006
static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
1007
1008
#endif /* CONFIG_HOTPLUG_CPU */

1009
static ssize_t dispatching_show(struct sysdev_class *class, char *buf)
1010
1011
1012
1013
1014
1015
1016
1017
1018
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", cpu_management);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

1019
1020
static ssize_t dispatching_store(struct sysdev_class *dev, const char *buf,
				 size_t count)
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
{
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
	rc = 0;
	get_online_cpus();
1031
	mutex_lock(&smp_cpu_state_mutex);
1032
1033
1034
1035
1036
1037
1038
	if (cpu_management == val)
		goto out;
	rc = topology_set_cpu_management(val);
	if (!rc)
		cpu_management = val;
out:
	mutex_unlock(&smp_cpu_state_mutex);
1039
	put_online_cpus();
1040
1041
	return rc ? rc : count;
}
1042
1043
static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
			 dispatching_store);
1044

Linus Torvalds's avatar
Linus Torvalds committed
1045
1046
1047
static int __init topology_init(void)
{
	int cpu;
1048
	int rc;
1049
1050

	register_cpu_notifier(&smp_cpu_nb);
Linus Torvalds's avatar
Linus Torvalds committed
1051

1052
#ifdef CONFIG_HOTPLUG_CPU
1053
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
1054
1055
1056
	if (rc)
		return rc;
#endif
1057
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
1058
1059
	if (rc)
		return rc;
1060
1061
	for_each_present_cpu(cpu) {
		rc = smp_add_present_cpu(cpu);
1062
1063
		if (rc)
			return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1064
1065
1066
1067
	}
	return 0;
}
subsys_initcall(topology_init);