smp.c 25 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  arch/s390/kernel/smp.c
 *
4
 *    Copyright IBM Corp. 1999,2007
Linus Torvalds's avatar
Linus Torvalds committed
5
 *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6
7
 *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
 *		 Heiko Carstens (heiko.carstens@de.ibm.com)
Linus Torvalds's avatar
Linus Torvalds committed
8
 *
9
 *  based on other smp stuff by
Linus Torvalds's avatar
Linus Torvalds committed
10
11
12
13
14
15
16
17
18
19
20
21
22
 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *    (c) 1998 Ingo Molnar
 *
 * We work with logical cpu numbering everywhere we can. The only
 * functions using the real cpu address (got from STAP) are the sigp
 * functions. For all other functions we use the identity mapping.
 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
 * used e.g. to find the idle task belonging to a logical cpu. Every array
 * in the kernel is sorted by the logical cpu number and not by the physical
 * one which is causing all the confusion with __cpu_logical_map and
 * cpu_number_map in other architectures.
 */

23
24
25
#define KMSG_COMPONENT "cpu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
26
27
28
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
29
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
31
32
33
34
35
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/cpu.h>
36
#include <linux/timex.h>
Michael Holzheu's avatar
Michael Holzheu committed
37
#include <linux/bootmem.h>
Michael Holzheu's avatar
Michael Holzheu committed
38
#include <asm/ipl.h>
39
#include <asm/setup.h>
Linus Torvalds's avatar
Linus Torvalds committed
40
41
42
43
44
45
#include <asm/sigp.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
#include <asm/s390_ext.h>
#include <asm/cpcmd.h>
#include <asm/tlbflush.h>
46
#include <asm/timer.h>
Michael Holzheu's avatar
Michael Holzheu committed
47
#include <asm/lowcore.h>
48
#include <asm/sclp.h>
49
#include <asm/cpu.h>
50
#include <asm/vdso.h>
51
#include "entry.h"
Linus Torvalds's avatar
Linus Torvalds committed
52
53
54

static struct task_struct *current_set[NR_CPUS];

55
56
57
58
59
60
61
62
static u8 smp_cpu_type;
static int smp_use_sigp_detection;

enum s390_cpu_state {
	CPU_STATE_STANDBY,
	CPU_STATE_CONFIGURED,
};

63
DEFINE_MUTEX(smp_cpu_state_mutex);
64
int smp_cpu_polarization[NR_CPUS];
65
static int smp_cpu_state[NR_CPUS];
66
static int cpu_management;
67
68
69

static DEFINE_PER_CPU(struct cpu, cpu_devices);

Linus Torvalds's avatar
Linus Torvalds committed
70
71
static void smp_ext_bitcall(int, ec_bit_sig);

72
void smp_send_stop(void)
Linus Torvalds's avatar
Linus Torvalds committed
73
{
74
	int cpu, rc;
Linus Torvalds's avatar
Linus Torvalds committed
75

76
77
	/* Disable all interrupts/machine checks */
	__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
Linus Torvalds's avatar
Linus Torvalds committed
78

79
	/* stop all processors */
Linus Torvalds's avatar
Linus Torvalds committed
80
81
82
83
	for_each_online_cpu(cpu) {
		if (cpu == smp_processor_id())
			continue;
		do {
84
			rc = signal_processor(cpu, sigp_stop);
85
		} while (rc == sigp_busy);
Linus Torvalds's avatar
Linus Torvalds committed
86

87
		while (!smp_cpu_not_running(cpu))
Heiko Carstens's avatar
Heiko Carstens committed
88
89
90
91
			cpu_relax();
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
92
93
94
95
96
/*
 * This is the main routine where commands issued by other
 * cpus are handled.
 */

97
static void do_ext_call_interrupt(__u16 code)
Linus Torvalds's avatar
Linus Torvalds committed
98
{
99
	unsigned long bits;
Linus Torvalds's avatar
Linus Torvalds committed
100

101
102
103
104
105
106
	/*
	 * handle bit signal external calls
	 *
	 * For the ec_schedule signal we have to do nothing. All the work
	 * is done automatically when we return from the interrupt.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
107
108
	bits = xchg(&S390_lowcore.ext_call_fast, 0);

109
	if (test_bit(ec_call_function, &bits))
110
111
112
113
		generic_smp_call_function_interrupt();

	if (test_bit(ec_call_function_single, &bits))
		generic_smp_call_function_single_interrupt();
Linus Torvalds's avatar
Linus Torvalds committed
114
115
116
117
118
119
120
121
}

/*
 * Send an external call sigp to another cpu and return without waiting
 * for its completion.
 */
static void smp_ext_bitcall(int cpu, ec_bit_sig sig)
{
122
123
124
	/*
	 * Set signaling bit in lowcore of target cpu and kick it
	 */
Linus Torvalds's avatar
Linus Torvalds committed
125
	set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
126
	while (signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds's avatar
Linus Torvalds committed
127
128
129
		udelay(10);
}

130
131
132
133
134
135
136
137
138
139
140
141
142
void arch_send_call_function_ipi(cpumask_t mask)
{
	int cpu;

	for_each_cpu_mask(cpu, mask)
		smp_ext_bitcall(cpu, ec_call_function);
}

void arch_send_call_function_single_ipi(int cpu)
{
	smp_ext_bitcall(cpu, ec_call_function_single);
}

143
#ifndef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
144
145
146
/*
 * this function sends a 'purge tlb' signal to another CPU.
 */
147
static void smp_ptlb_callback(void *info)
Linus Torvalds's avatar
Linus Torvalds committed
148
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
149
	__tlb_flush_local();
Linus Torvalds's avatar
Linus Torvalds committed
150
151
152
153
}

void smp_ptlb_all(void)
{
154
	on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds's avatar
Linus Torvalds committed
155
156
}
EXPORT_SYMBOL(smp_ptlb_all);
157
#endif /* ! CONFIG_64BIT */
Linus Torvalds's avatar
Linus Torvalds committed
158
159
160
161
162
163
164
165

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
void smp_send_reschedule(int cpu)
{
166
	smp_ext_bitcall(cpu, ec_schedule);
Linus Torvalds's avatar
Linus Torvalds committed
167
168
169
170
171
}

/*
 * parameter area for the set/clear control bit callbacks
 */
172
struct ec_creg_mask_parms {
Linus Torvalds's avatar
Linus Torvalds committed
173
174
	unsigned long orvals[16];
	unsigned long andvals[16];
175
};
Linus Torvalds's avatar
Linus Torvalds committed
176
177
178
179

/*
 * callback for setting/clearing control bits
 */
180
181
static void smp_ctl_bit_callback(void *info)
{
182
	struct ec_creg_mask_parms *pp = info;
Linus Torvalds's avatar
Linus Torvalds committed
183
184
	unsigned long cregs[16];
	int i;
185

186
187
	__ctl_store(cregs, 0, 15);
	for (i = 0; i <= 15; i++)
Linus Torvalds's avatar
Linus Torvalds committed
188
		cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
189
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
190
191
192
193
194
}

/*
 * Set a bit in a control register of all cpus
 */
195
196
197
void smp_ctl_set_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
198

199
200
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
201
	parms.orvals[cr] = 1 << bit;
202
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
203
}
204
EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds's avatar
Linus Torvalds committed
205
206
207
208

/*
 * Clear a bit in a control register of all cpus
 */
209
210
211
void smp_ctl_clear_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
212

213
214
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
215
	parms.andvals[cr] = ~(1L << bit);
216
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
217
}
218
EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds's avatar
Linus Torvalds committed
219

220
221
222
223
224
225
226
/*
 * In early ipl state a temp. logically cpu number is needed, so the sigp
 * functions can be used to sense other cpus. Since NR_CPUS is >= 2 on
 * CONFIG_SMP and the ipl cpu is logical cpu 0, it must be 1.
 */
#define CPU_INIT_NO	1

Michael Holzheu's avatar
Michael Holzheu committed
227
228
229
230
231
232
233
234
235
236
237
#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_ZFCPDUMP_MODULE)

/*
 * zfcpdump_prefix_array holds prefix registers for the following scenario:
 * 64 bit zfcpdump kernel and 31 bit kernel which is to be dumped. We have to
 * save its prefix registers, since they get lost, when switching from 31 bit
 * to 64 bit.
 */
unsigned int zfcpdump_prefix_array[NR_CPUS + 1] \
	__attribute__((__section__(".data")));

238
static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
Michael Holzheu's avatar
Michael Holzheu committed
239
240
241
{
	if (ipl_info.type != IPL_TYPE_FCP_DUMP)
		return;
242
	if (cpu >= NR_CPUS) {
243
244
		pr_warning("CPU %i exceeds the maximum %i and is excluded from "
			   "the dump\n", cpu, NR_CPUS - 1);
245
		return;
Michael Holzheu's avatar
Michael Holzheu committed
246
	}
247
	zfcpdump_save_areas[cpu] = kmalloc(sizeof(union save_area), GFP_KERNEL);
248
249
250
	__cpu_logical_map[CPU_INIT_NO] = (__u16) phy_cpu;
	while (signal_processor(CPU_INIT_NO, sigp_stop_and_store_status) ==
	       sigp_busy)
251
252
253
254
255
256
257
258
		cpu_relax();
	memcpy(zfcpdump_save_areas[cpu],
	       (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
	       SAVE_AREA_SIZE);
#ifdef CONFIG_64BIT
	/* copy original prefix register */
	zfcpdump_save_areas[cpu]->s390x.pref_reg = zfcpdump_prefix_array[cpu];
#endif
Michael Holzheu's avatar
Michael Holzheu committed
259
260
261
262
263
264
}

union save_area *zfcpdump_save_areas[NR_CPUS + 1];
EXPORT_SYMBOL_GPL(zfcpdump_save_areas);

#else
265
266
267
268

static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }

#endif /* CONFIG_ZFCPDUMP || CONFIG_ZFCPDUMP_MODULE */
Michael Holzheu's avatar
Michael Holzheu committed
269

270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
static int cpu_stopped(int cpu)
{
	__u32 status;

	/* Check for stopped state */
	if (signal_processor_ps(&status, 0, cpu, sigp_sense) ==
	    sigp_status_stored) {
		if (status & 0x40)
			return 1;
	}
	return 0;
}

static int cpu_known(int cpu_id)
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (__cpu_logical_map[cpu] == cpu_id)
			return 1;
	}
	return 0;
}

static int smp_rescan_cpus_sigp(cpumask_t avail)
{
	int cpu_id, logical_cpu;

	logical_cpu = first_cpu(avail);
	if (logical_cpu == NR_CPUS)
		return 0;
	for (cpu_id = 0; cpu_id <= 65535; cpu_id++) {
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
305
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
306
307
308
309
310
311
312
313
314
315
316
		if (!cpu_stopped(logical_cpu))
			continue;
		cpu_set(logical_cpu, cpu_present_map);
		smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
		logical_cpu = next_cpu(logical_cpu, avail);
		if (logical_cpu == NR_CPUS)
			break;
	}
	return 0;
}

317
static int smp_rescan_cpus_sclp(cpumask_t avail)
318
319
320
321
322
323
324
325
{
	struct sclp_cpu_info *info;
	int cpu_id, logical_cpu, cpu;
	int rc;

	logical_cpu = first_cpu(avail);
	if (logical_cpu == NR_CPUS)
		return 0;
326
	info = kmalloc(sizeof(*info), GFP_KERNEL);
327
328
329
330
331
332
333
334
335
336
337
338
	if (!info)
		return -ENOMEM;
	rc = sclp_get_cpu_info(info);
	if (rc)
		goto out;
	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_id = info->cpu[cpu].address;
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
339
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
340
341
342
343
344
345
346
347
348
349
		cpu_set(logical_cpu, cpu_present_map);
		if (cpu >= info->configured)
			smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
		else
			smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
		logical_cpu = next_cpu(logical_cpu, avail);
		if (logical_cpu == NR_CPUS)
			break;
	}
out:
350
	kfree(info);
351
352
353
	return rc;
}

354
static int __smp_rescan_cpus(void)
355
356
357
{
	cpumask_t avail;

358
	cpus_xor(avail, cpu_possible_map, cpu_present_map);
359
360
361
362
	if (smp_use_sigp_detection)
		return smp_rescan_cpus_sigp(avail);
	else
		return smp_rescan_cpus_sclp(avail);
Linus Torvalds's avatar
Linus Torvalds committed
363
364
}

365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
static void __init smp_detect_cpus(void)
{
	unsigned int cpu, c_cpus, s_cpus;
	struct sclp_cpu_info *info;
	u16 boot_cpu_addr, cpu_addr;

	c_cpus = 1;
	s_cpus = 0;
	boot_cpu_addr = S390_lowcore.cpu_data.cpu_addr;
	info = kmalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		panic("smp_detect_cpus failed to allocate memory\n");
	/* Use sigp detection algorithm if sclp doesn't work. */
	if (sclp_get_cpu_info(info)) {
		smp_use_sigp_detection = 1;
		for (cpu = 0; cpu <= 65535; cpu++) {
			if (cpu == boot_cpu_addr)
				continue;
			__cpu_logical_map[CPU_INIT_NO] = cpu;
			if (!cpu_stopped(CPU_INIT_NO))
				continue;
			smp_get_save_area(c_cpus, cpu);
			c_cpus++;
		}
		goto out;
	}

	if (info->has_cpu_type) {
		for (cpu = 0; cpu < info->combined; cpu++) {
			if (info->cpu[cpu].address == boot_cpu_addr) {
				smp_cpu_type = info->cpu[cpu].type;
				break;
			}
		}
	}

	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_addr = info->cpu[cpu].address;
		if (cpu_addr == boot_cpu_addr)
			continue;
		__cpu_logical_map[CPU_INIT_NO] = cpu_addr;
		if (!cpu_stopped(CPU_INIT_NO)) {
			s_cpus++;
			continue;
		}
		smp_get_save_area(c_cpus, cpu_addr);
		c_cpus++;
	}
out:
	kfree(info);
417
	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
418
	get_online_cpus();
419
	__smp_rescan_cpus();
420
	put_online_cpus();
421
422
}

Linus Torvalds's avatar
Linus Torvalds committed
423
/*
424
 *	Activate a secondary processor.
Linus Torvalds's avatar
Linus Torvalds committed
425
 */
426
int __cpuinit start_secondary(void *cpuvoid)
Linus Torvalds's avatar
Linus Torvalds committed
427
{
428
429
	/* Setup the cpu */
	cpu_init();
430
	preempt_disable();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
431
	/* Enable TOD clock interrupts on the secondary cpu. */
432
	init_cpu_timer();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
433
	/* Enable cpu timer interrupts on the secondary cpu. */
434
	init_cpu_vtimer();
Linus Torvalds's avatar
Linus Torvalds committed
435
	/* Enable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
436
437
	pfault_init();

438
439
	/* call cpu notifiers */
	notify_cpu_starting(smp_processor_id());
Linus Torvalds's avatar
Linus Torvalds committed
440
	/* Mark this cpu as online */
441
	ipi_call_lock();
Linus Torvalds's avatar
Linus Torvalds committed
442
	cpu_set(smp_processor_id(), cpu_online_map);
443
	ipi_call_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
444
445
	/* Switch on interrupts */
	local_irq_enable();
446
447
448
449
450
	/* Print info about this processor */
	print_cpu_info(&S390_lowcore.cpu_data);
	/* cpu_idle will call schedule for us */
	cpu_idle();
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
}

static void __init smp_create_idle(unsigned int cpu)
{
	struct task_struct *p;

	/*
	 *  don't care about the psw and regs settings since we'll never
	 *  reschedule the forked task.
	 */
	p = fork_idle(cpu);
	if (IS_ERR(p))
		panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
	current_set[cpu] = p;
}

467
468
469
470
471
472
473
474
475
476
477
478
static int __cpuinit smp_alloc_lowcore(int cpu)
{
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	if (!lowcore)
		return -ENOMEM;
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
	panic_stack = __get_free_page(GFP_KERNEL);
479
480
	if (!panic_stack || !async_stack)
		goto out;
481
482
	memcpy(lowcore, &S390_lowcore, 512);
	memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
483
484
485
486
487
488
489
490
491
	lowcore->async_stack = async_stack + ASYNC_SIZE;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;

#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		unsigned long save_area;

		save_area = get_zeroed_page(GFP_KERNEL);
		if (!save_area)
492
			goto out;
493
494
		lowcore->extended_save_area_addr = (u32) save_area;
	}
495
496
497
#else
	if (vdso_alloc_per_cpu(cpu, lowcore))
		goto out;
498
499
500
501
#endif
	lowcore_ptr[cpu] = lowcore;
	return 0;

502
out:
503
	free_page(panic_stack);
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
	free_pages(async_stack, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	return -ENOMEM;
}

#ifdef CONFIG_HOTPLUG_CPU
static void smp_free_lowcore(int cpu)
{
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = lowcore_ptr[cpu];
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		free_page((unsigned long) lowcore->extended_save_area_addr);
520
521
#else
	vdso_free_per_cpu(cpu, lowcore);
522
523
524
525
526
527
528
529
#endif
	free_page(lowcore->panic_stack - PAGE_SIZE);
	free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	lowcore_ptr[cpu] = NULL;
}
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
530
/* Upping and downing of CPUs */
531
int __cpuinit __cpu_up(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
532
533
{
	struct task_struct *idle;
534
	struct _lowcore *cpu_lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
535
	struct stack_frame *sf;
536
	sigp_ccode ccode;
Linus Torvalds's avatar
Linus Torvalds committed
537

538
539
	if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
		return -EIO;
540
541
	if (smp_alloc_lowcore(cpu))
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
542
543
544

	ccode = signal_processor_p((__u32)(unsigned long)(lowcore_ptr[cpu]),
				   cpu, sigp_set_prefix);
545
	if (ccode)
Linus Torvalds's avatar
Linus Torvalds committed
546
547
548
		return -EIO;

	idle = current_set[cpu];
549
	cpu_lowcore = lowcore_ptr[cpu];
Linus Torvalds's avatar
Linus Torvalds committed
550
	cpu_lowcore->kernel_stack = (unsigned long)
551
		task_stack_page(idle) + THREAD_SIZE;
552
	cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
Linus Torvalds's avatar
Linus Torvalds committed
553
554
555
556
557
558
	sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
				     - sizeof(struct pt_regs)
				     - sizeof(struct stack_frame));
	memset(sf, 0, sizeof(struct stack_frame));
	sf->gprs[9] = (unsigned long) sf;
	cpu_lowcore->save_area[15] = (unsigned long) sf;
559
	__ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
560
561
562
	asm volatile(
		"	stam	0,15,0(%0)"
		: : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
Linus Torvalds's avatar
Linus Torvalds committed
563
	cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
564
565
	cpu_lowcore->current_task = (unsigned long) idle;
	cpu_lowcore->cpu_data.cpu_nr = cpu;
566
567
	cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
	cpu_lowcore->ipl_device = S390_lowcore.ipl_device;
Linus Torvalds's avatar
Linus Torvalds committed
568
	eieio();
Michael Ryan's avatar
Michael Ryan committed
569

570
	while (signal_processor(cpu, sigp_restart) == sigp_busy)
Michael Ryan's avatar
Michael Ryan committed
571
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
572
573
574
575
576
577

	while (!cpu_online(cpu))
		cpu_relax();
	return 0;
}

578
static int __init setup_possible_cpus(char *s)
579
{
580
	int pcpus, cpu;
581

582
583
584
	pcpus = simple_strtoul(s, NULL, 0);
	cpu_possible_map = cpumask_of_cpu(0);
	for (cpu = 1; cpu < pcpus && cpu < NR_CPUS; cpu++)
585
		cpu_set(cpu, cpu_possible_map);
586
587
588
589
	return 0;
}
early_param("possible_cpus", setup_possible_cpus);

590
591
#ifdef CONFIG_HOTPLUG_CPU

592
int __cpu_disable(void)
Linus Torvalds's avatar
Linus Torvalds committed
593
{
594
	struct ec_creg_mask_parms cr_parms;
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
595
	int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
596

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
597
	cpu_clear(cpu, cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
598
599

	/* Disable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
600
	pfault_fini();
Linus Torvalds's avatar
Linus Torvalds committed
601

602
603
	memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
	memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
604

605
	/* disable all external interrupts */
Linus Torvalds's avatar
Linus Torvalds committed
606
	cr_parms.orvals[0] = 0;
607
608
	cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
				1 << 11 | 1 << 10 | 1 <<  6 | 1 <<  4);
Linus Torvalds's avatar
Linus Torvalds committed
609
610
	/* disable all I/O interrupts */
	cr_parms.orvals[6] = 0;
611
612
	cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
				1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
Linus Torvalds's avatar
Linus Torvalds committed
613
614
	/* disable most machine checks */
	cr_parms.orvals[14] = 0;
615
616
	cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
				 1 << 25 | 1 << 24);
617

Linus Torvalds's avatar
Linus Torvalds committed
618
619
620
621
622
	smp_ctl_bit_callback(&cr_parms);

	return 0;
}

623
void __cpu_die(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
624
625
626
627
{
	/* Wait until target cpu is down */
	while (!smp_cpu_not_running(cpu))
		cpu_relax();
628
	smp_free_lowcore(cpu);
629
	pr_info("Processor %d stopped\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
630
631
}

632
void cpu_die(void)
Linus Torvalds's avatar
Linus Torvalds committed
633
634
635
636
{
	idle_task_exit();
	signal_processor(smp_processor_id(), sigp_stop);
	BUG();
637
	for (;;);
Linus Torvalds's avatar
Linus Torvalds committed
638
639
}

640
641
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
642
643
void __init smp_prepare_cpus(unsigned int max_cpus)
{
644
645
646
647
648
#ifndef CONFIG_64BIT
	unsigned long save_area = 0;
#endif
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
649
	unsigned int cpu;
650
	int lc_order;
651

652
653
	smp_detect_cpus();

654
655
656
	/* request the 0x1201 emergency signal external interrupt */
	if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1201");
Linus Torvalds's avatar
Linus Torvalds committed
657
658
	print_cpu_info(&S390_lowcore.cpu_data);

659
660
661
662
663
	/* Reallocate current lowcore, but keep its contents. */
	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	panic_stack = __get_free_page(GFP_KERNEL);
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
664
	BUG_ON(!lowcore || !panic_stack || !async_stack);
665
#ifndef CONFIG_64BIT
666
	if (MACHINE_HAS_IEEE)
667
		save_area = get_zeroed_page(GFP_KERNEL);
668
#endif
669
670
671
672
673
674
675
676
677
	local_irq_disable();
	local_mcck_disable();
	lowcore_ptr[smp_processor_id()] = lowcore;
	*lowcore = S390_lowcore;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;
	lowcore->async_stack = async_stack + ASYNC_SIZE;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		lowcore->extended_save_area_addr = (u32) save_area;
678
#else
Heiko Carstens's avatar
Heiko Carstens committed
679
680
	if (vdso_alloc_per_cpu(smp_processor_id(), lowcore))
		BUG();
681
682
683
684
#endif
	set_prefix((u32)(unsigned long) lowcore);
	local_mcck_enable();
	local_irq_enable();
685
	for_each_possible_cpu(cpu)
Linus Torvalds's avatar
Linus Torvalds committed
686
687
688
689
		if (cpu != smp_processor_id())
			smp_create_idle(cpu);
}

690
void __init smp_prepare_boot_cpu(void)
Linus Torvalds's avatar
Linus Torvalds committed
691
692
693
{
	BUG_ON(smp_processor_id() != 0);

694
695
	current_thread_info()->cpu = 0;
	cpu_set(0, cpu_present_map);
Linus Torvalds's avatar
Linus Torvalds committed
696
697
698
	cpu_set(0, cpu_online_map);
	S390_lowcore.percpu_offset = __per_cpu_offset[0];
	current_set[0] = current;
699
	smp_cpu_state[0] = CPU_STATE_CONFIGURED;
700
	smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
Linus Torvalds's avatar
Linus Torvalds committed
701
702
}

703
void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds's avatar
Linus Torvalds committed
704
705
706
707
708
709
710
711
712
713
714
{
}

/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
715
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
716
717
}

718
#ifdef CONFIG_HOTPLUG_CPU
719
720
static ssize_t cpu_configure_show(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
721
722
723
724
725
726
727
728
729
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

730
731
732
static ssize_t cpu_configure_store(struct sys_device *dev,
				  struct sysdev_attribute *attr,
				  const char *buf, size_t count)
733
734
735
736
737
738
739
740
741
742
{
	int cpu = dev->id;
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;

743
	get_online_cpus();
744
	mutex_lock(&smp_cpu_state_mutex);
745
746
747
748
749
750
751
752
	rc = -EBUSY;
	if (cpu_online(cpu))
		goto out;
	rc = 0;
	switch (val) {
	case 0:
		if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
			rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
753
			if (!rc) {
754
				smp_cpu_state[cpu] = CPU_STATE_STANDBY;
755
756
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
757
758
759
760
761
		}
		break;
	case 1:
		if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
			rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
762
			if (!rc) {
763
				smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
764
765
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
766
767
768
769
770
771
772
		}
		break;
	default:
		break;
	}
out:
	mutex_unlock(&smp_cpu_state_mutex);
773
	put_online_cpus();
774
775
776
777
778
	return rc ? rc : count;
}
static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
#endif /* CONFIG_HOTPLUG_CPU */

779
780
static ssize_t cpu_polarization_show(struct sys_device *dev,
				     struct sysdev_attribute *attr, char *buf)
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	switch (smp_cpu_polarization[cpu]) {
	case POLARIZATION_HRZ:
		count = sprintf(buf, "horizontal\n");
		break;
	case POLARIZATION_VL:
		count = sprintf(buf, "vertical:low\n");
		break;
	case POLARIZATION_VM:
		count = sprintf(buf, "vertical:medium\n");
		break;
	case POLARIZATION_VH:
		count = sprintf(buf, "vertical:high\n");
		break;
	default:
		count = sprintf(buf, "unknown\n");
		break;
	}
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);

808
809
static ssize_t show_cpu_address(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
810
811
812
813
814
815
816
817
818
819
820
{
	return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
}
static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);


static struct attribute *cpu_common_attrs[] = {
#ifdef CONFIG_HOTPLUG_CPU
	&attr_configure.attr,
#endif
	&attr_address.attr,
821
	&attr_polarization.attr,
822
823
824
825
826
827
	NULL,
};

static struct attribute_group cpu_common_attr_group = {
	.attrs = cpu_common_attrs,
};
Linus Torvalds's avatar
Linus Torvalds committed
828

829
830
static ssize_t show_capability(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
831
832
833
834
835
836
837
838
839
840
841
{
	unsigned int capability;
	int rc;

	rc = get_cpu_capability(&capability);
	if (rc)
		return rc;
	return sprintf(buf, "%u\n", capability);
}
static SYSDEV_ATTR(capability, 0444, show_capability, NULL);

842
843
static ssize_t show_idle_count(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
844
845
846
847
848
{
	struct s390_idle_data *idle;
	unsigned long long idle_count;

	idle = &per_cpu(s390_idle, dev->id);
849
	spin_lock(&idle->lock);
850
	idle_count = idle->idle_count;
851
852
853
	if (idle->idle_enter)
		idle_count++;
	spin_unlock(&idle->lock);
854
855
856
857
	return sprintf(buf, "%llu\n", idle_count);
}
static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);

858
859
static ssize_t show_idle_time(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
860
861
{
	struct s390_idle_data *idle;
862
	unsigned long long now, idle_time, idle_enter;
863
864

	idle = &per_cpu(s390_idle, dev->id);
865
866
867
868
869
870
871
872
	spin_lock(&idle->lock);
	now = get_clock();
	idle_time = idle->idle_time;
	idle_enter = idle->idle_enter;
	if (idle_enter != 0ULL && idle_enter < now)
		idle_time += now - idle_enter;
	spin_unlock(&idle->lock);
	return sprintf(buf, "%llu\n", idle_time >> 12);
873
}
874
static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
875

876
static struct attribute *cpu_online_attrs[] = {
877
878
	&attr_capability.attr,
	&attr_idle_count.attr,
879
	&attr_idle_time_us.attr,
880
881
882
	NULL,
};

883
884
static struct attribute_group cpu_online_attr_group = {
	.attrs = cpu_online_attrs,
885
886
};

887
888
889
890
891
892
static int __cpuinit smp_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned int)(long)hcpu;
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
893
	struct s390_idle_data *idle;
894
895
896

	switch (action) {
	case CPU_ONLINE:
897
	case CPU_ONLINE_FROZEN:
898
899
900
901
902
903
		idle = &per_cpu(s390_idle, cpu);
		spin_lock_irq(&idle->lock);
		idle->idle_enter = 0;
		idle->idle_time = 0;
		idle->idle_count = 0;
		spin_unlock_irq(&idle->lock);
904
		if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
905
906
907
			return NOTIFY_BAD;
		break;
	case CPU_DEAD:
908
	case CPU_DEAD_FROZEN:
909
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
910
911
912
913
914
915
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata smp_cpu_nb = {
916
	.notifier_call = smp_cpu_notify,
917
918
};

919
static int __devinit smp_add_present_cpu(int cpu)
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
{
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
	int rc;

	c->hotpluggable = 1;
	rc = register_cpu(c, cpu);
	if (rc)
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
	if (rc)
		goto out_cpu;
	if (!cpu_online(cpu))
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
	if (!rc)
		return 0;
	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
out_cpu:
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu(c);
#endif
out:
	return rc;
}

#ifdef CONFIG_HOTPLUG_CPU
947

948
int __ref smp_rescan_cpus(void)
949
950
951
952
953
{
	cpumask_t newcpus;
	int cpu;
	int rc;

954
	get_online_cpus();
955
	mutex_lock(&smp_cpu_state_mutex);
956
	newcpus = cpu_present_map;
957
	rc = __smp_rescan_cpus();
958
959
960
961
962
963
964
965
966
967
968
	if (rc)
		goto out;
	cpus_andnot(newcpus, cpu_present_map, newcpus);
	for_each_cpu_mask(cpu, newcpus) {
		rc = smp_add_present_cpu(cpu);
		if (rc)
			cpu_clear(cpu, cpu_present_map);
	}
	rc = 0;
out:
	mutex_unlock(&smp_cpu_state_mutex);
969
	put_online_cpus();
970
971
	if (!cpus_empty(newcpus))
		topology_schedule_update();
972
973
974
	return rc;
}

975
static ssize_t __ref rescan_store(struct sysdev_class *class, const char *buf,
976
977
978
979
980
				  size_t count)
{
	int rc;

	rc = smp_rescan_cpus();
981
982
	return rc ? rc : count;
}
983
static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
984
985
#endif /* CONFIG_HOTPLUG_CPU */

986
static ssize_t dispatching_show(struct sysdev_class *class, char *buf)
987
988
989
990
991
992
993
994
995
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", cpu_management);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

996
997
static ssize_t dispatching_store(struct sysdev_class *dev, const char *buf,
				 size_t count)
998
999
1000
1001
1002
1003
1004
1005
1006
1007
{
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
	rc = 0;
	get_online_cpus();
1008
	mutex_lock(&smp_cpu_state_mutex);
1009
1010
1011
1012
1013
1014
1015
	if (cpu_management == val)
		goto out;
	rc = topology_set_cpu_management(val);
	if (!rc)
		cpu_management = val;
out:
	mutex_unlock(&smp_cpu_state_mutex);
1016
	put_online_cpus();
1017
1018
	return rc ? rc : count;
}
1019
1020
static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
			 dispatching_store);
1021

Linus Torvalds's avatar
Linus Torvalds committed
1022
1023
1024
static int __init topology_init(void)
{
	int cpu;
1025
	int rc;
1026
1027

	register_cpu_notifier(&smp_cpu_nb);
Linus Torvalds's avatar
Linus Torvalds committed
1028

1029
#ifdef CONFIG_HOTPLUG_CPU
1030
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
1031
1032
1033
	if (rc)
		return rc;
#endif
1034
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
1035
1036
	if (rc)
		return rc;
1037
1038
	for_each_present_cpu(cpu) {
		rc = smp_add_present_cpu(cpu);
1039
1040
		if (rc)
			return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1041
1042
1043
1044
	}
	return 0;
}
subsys_initcall(topology_init);