smp.c 25.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  arch/s390/kernel/smp.c
 *
4
 *    Copyright IBM Corp. 1999,2007
Linus Torvalds's avatar
Linus Torvalds committed
5
 *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6
7
 *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
 *		 Heiko Carstens (heiko.carstens@de.ibm.com)
Linus Torvalds's avatar
Linus Torvalds committed
8
 *
9
 *  based on other smp stuff by
Linus Torvalds's avatar
Linus Torvalds committed
10
11
12
13
14
15
16
17
18
19
20
21
22
 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *    (c) 1998 Ingo Molnar
 *
 * We work with logical cpu numbering everywhere we can. The only
 * functions using the real cpu address (got from STAP) are the sigp
 * functions. For all other functions we use the identity mapping.
 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
 * used e.g. to find the idle task belonging to a logical cpu. Every array
 * in the kernel is sorted by the logical cpu number and not by the physical
 * one which is causing all the confusion with __cpu_logical_map and
 * cpu_number_map in other architectures.
 */

23
24
25
#define KMSG_COMPONENT "cpu"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
26
27
28
#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
29
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
30
31
32
33
34
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
35
#include <linux/irqflags.h>
Linus Torvalds's avatar
Linus Torvalds committed
36
#include <linux/cpu.h>
37
#include <linux/timex.h>
Michael Holzheu's avatar
Michael Holzheu committed
38
#include <linux/bootmem.h>
Michael Holzheu's avatar
Michael Holzheu committed
39
#include <asm/ipl.h>
40
#include <asm/setup.h>
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
#include <asm/sigp.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
#include <asm/s390_ext.h>
#include <asm/cpcmd.h>
#include <asm/tlbflush.h>
47
#include <asm/timer.h>
Michael Holzheu's avatar
Michael Holzheu committed
48
#include <asm/lowcore.h>
49
#include <asm/sclp.h>
50
#include <asm/cpu.h>
51
#include <asm/vdso.h>
52
#include "entry.h"
Linus Torvalds's avatar
Linus Torvalds committed
53
54
55

static struct task_struct *current_set[NR_CPUS];

56
57
58
59
60
61
62
63
static u8 smp_cpu_type;
static int smp_use_sigp_detection;

enum s390_cpu_state {
	CPU_STATE_STANDBY,
	CPU_STATE_CONFIGURED,
};

64
DEFINE_MUTEX(smp_cpu_state_mutex);
65
int smp_cpu_polarization[NR_CPUS];
66
static int smp_cpu_state[NR_CPUS];
67
static int cpu_management;
68
69
70

static DEFINE_PER_CPU(struct cpu, cpu_devices);

Linus Torvalds's avatar
Linus Torvalds committed
71
72
static void smp_ext_bitcall(int, ec_bit_sig);

73
void smp_send_stop(void)
Linus Torvalds's avatar
Linus Torvalds committed
74
{
75
	int cpu, rc;
Linus Torvalds's avatar
Linus Torvalds committed
76

77
78
	/* Disable all interrupts/machine checks */
	__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
79
	trace_hardirqs_off();
Linus Torvalds's avatar
Linus Torvalds committed
80

81
	/* stop all processors */
Linus Torvalds's avatar
Linus Torvalds committed
82
83
84
85
	for_each_online_cpu(cpu) {
		if (cpu == smp_processor_id())
			continue;
		do {
86
			rc = signal_processor(cpu, sigp_stop);
87
		} while (rc == sigp_busy);
Linus Torvalds's avatar
Linus Torvalds committed
88

89
		while (!smp_cpu_not_running(cpu))
Heiko Carstens's avatar
Heiko Carstens committed
90
91
92
93
			cpu_relax();
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
94
95
96
97
98
/*
 * This is the main routine where commands issued by other
 * cpus are handled.
 */

99
static void do_ext_call_interrupt(__u16 code)
Linus Torvalds's avatar
Linus Torvalds committed
100
{
101
	unsigned long bits;
Linus Torvalds's avatar
Linus Torvalds committed
102

103
104
105
106
107
108
	/*
	 * handle bit signal external calls
	 *
	 * For the ec_schedule signal we have to do nothing. All the work
	 * is done automatically when we return from the interrupt.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
109
110
	bits = xchg(&S390_lowcore.ext_call_fast, 0);

111
	if (test_bit(ec_call_function, &bits))
112
113
114
115
		generic_smp_call_function_interrupt();

	if (test_bit(ec_call_function_single, &bits))
		generic_smp_call_function_single_interrupt();
Linus Torvalds's avatar
Linus Torvalds committed
116
117
118
119
120
121
122
123
}

/*
 * Send an external call sigp to another cpu and return without waiting
 * for its completion.
 */
static void smp_ext_bitcall(int cpu, ec_bit_sig sig)
{
124
125
126
	/*
	 * Set signaling bit in lowcore of target cpu and kick it
	 */
Linus Torvalds's avatar
Linus Torvalds committed
127
	set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
128
	while (signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds's avatar
Linus Torvalds committed
129
130
131
		udelay(10);
}

132
133
134
135
136
137
138
139
140
141
142
143
144
void arch_send_call_function_ipi(cpumask_t mask)
{
	int cpu;

	for_each_cpu_mask(cpu, mask)
		smp_ext_bitcall(cpu, ec_call_function);
}

void arch_send_call_function_single_ipi(int cpu)
{
	smp_ext_bitcall(cpu, ec_call_function_single);
}

145
#ifndef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
146
147
148
/*
 * this function sends a 'purge tlb' signal to another CPU.
 */
149
static void smp_ptlb_callback(void *info)
Linus Torvalds's avatar
Linus Torvalds committed
150
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
151
	__tlb_flush_local();
Linus Torvalds's avatar
Linus Torvalds committed
152
153
154
155
}

void smp_ptlb_all(void)
{
156
	on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds's avatar
Linus Torvalds committed
157
158
}
EXPORT_SYMBOL(smp_ptlb_all);
159
#endif /* ! CONFIG_64BIT */
Linus Torvalds's avatar
Linus Torvalds committed
160
161
162
163
164
165
166
167

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
void smp_send_reschedule(int cpu)
{
168
	smp_ext_bitcall(cpu, ec_schedule);
Linus Torvalds's avatar
Linus Torvalds committed
169
170
171
172
173
}

/*
 * parameter area for the set/clear control bit callbacks
 */
174
struct ec_creg_mask_parms {
Linus Torvalds's avatar
Linus Torvalds committed
175
176
	unsigned long orvals[16];
	unsigned long andvals[16];
177
};
Linus Torvalds's avatar
Linus Torvalds committed
178
179
180
181

/*
 * callback for setting/clearing control bits
 */
182
183
static void smp_ctl_bit_callback(void *info)
{
184
	struct ec_creg_mask_parms *pp = info;
Linus Torvalds's avatar
Linus Torvalds committed
185
186
	unsigned long cregs[16];
	int i;
187

188
189
	__ctl_store(cregs, 0, 15);
	for (i = 0; i <= 15; i++)
Linus Torvalds's avatar
Linus Torvalds committed
190
		cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
191
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
192
193
194
195
196
}

/*
 * Set a bit in a control register of all cpus
 */
197
198
199
void smp_ctl_set_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
200

201
202
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
203
	parms.orvals[cr] = 1 << bit;
204
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
205
}
206
EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds's avatar
Linus Torvalds committed
207
208
209
210

/*
 * Clear a bit in a control register of all cpus
 */
211
212
213
void smp_ctl_clear_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
214

215
216
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
217
	parms.andvals[cr] = ~(1L << bit);
218
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
219
}
220
EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds's avatar
Linus Torvalds committed
221

222
223
224
225
226
227
228
/*
 * In early ipl state a temp. logically cpu number is needed, so the sigp
 * functions can be used to sense other cpus. Since NR_CPUS is >= 2 on
 * CONFIG_SMP and the ipl cpu is logical cpu 0, it must be 1.
 */
#define CPU_INIT_NO	1

229
#ifdef CONFIG_ZFCPDUMP
Michael Holzheu's avatar
Michael Holzheu committed
230
231
232
233
234
235
236
237
238
239

/*
 * zfcpdump_prefix_array holds prefix registers for the following scenario:
 * 64 bit zfcpdump kernel and 31 bit kernel which is to be dumped. We have to
 * save its prefix registers, since they get lost, when switching from 31 bit
 * to 64 bit.
 */
unsigned int zfcpdump_prefix_array[NR_CPUS + 1] \
	__attribute__((__section__(".data")));

240
static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
Michael Holzheu's avatar
Michael Holzheu committed
241
242
243
{
	if (ipl_info.type != IPL_TYPE_FCP_DUMP)
		return;
244
	if (cpu >= NR_CPUS) {
245
246
		pr_warning("CPU %i exceeds the maximum %i and is excluded from "
			   "the dump\n", cpu, NR_CPUS - 1);
247
		return;
Michael Holzheu's avatar
Michael Holzheu committed
248
	}
249
	zfcpdump_save_areas[cpu] = kmalloc(sizeof(union save_area), GFP_KERNEL);
250
251
252
	__cpu_logical_map[CPU_INIT_NO] = (__u16) phy_cpu;
	while (signal_processor(CPU_INIT_NO, sigp_stop_and_store_status) ==
	       sigp_busy)
253
254
255
256
257
258
259
260
		cpu_relax();
	memcpy(zfcpdump_save_areas[cpu],
	       (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
	       SAVE_AREA_SIZE);
#ifdef CONFIG_64BIT
	/* copy original prefix register */
	zfcpdump_save_areas[cpu]->s390x.pref_reg = zfcpdump_prefix_array[cpu];
#endif
Michael Holzheu's avatar
Michael Holzheu committed
261
262
263
264
265
266
}

union save_area *zfcpdump_save_areas[NR_CPUS + 1];
EXPORT_SYMBOL_GPL(zfcpdump_save_areas);

#else
267
268
269

static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }

270
#endif /* CONFIG_ZFCPDUMP */
Michael Holzheu's avatar
Michael Holzheu committed
271

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
static int cpu_stopped(int cpu)
{
	__u32 status;

	/* Check for stopped state */
	if (signal_processor_ps(&status, 0, cpu, sigp_sense) ==
	    sigp_status_stored) {
		if (status & 0x40)
			return 1;
	}
	return 0;
}

static int cpu_known(int cpu_id)
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (__cpu_logical_map[cpu] == cpu_id)
			return 1;
	}
	return 0;
}

static int smp_rescan_cpus_sigp(cpumask_t avail)
{
	int cpu_id, logical_cpu;

300
301
	logical_cpu = cpumask_first(&avail);
	if (logical_cpu >= nr_cpu_ids)
302
303
304
305
306
		return 0;
	for (cpu_id = 0; cpu_id <= 65535; cpu_id++) {
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
307
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
308
309
310
311
		if (!cpu_stopped(logical_cpu))
			continue;
		cpu_set(logical_cpu, cpu_present_map);
		smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
312
313
		logical_cpu = cpumask_next(logical_cpu, &avail);
		if (logical_cpu >= nr_cpu_ids)
314
315
316
317
318
			break;
	}
	return 0;
}

319
static int smp_rescan_cpus_sclp(cpumask_t avail)
320
321
322
323
324
{
	struct sclp_cpu_info *info;
	int cpu_id, logical_cpu, cpu;
	int rc;

325
326
	logical_cpu = cpumask_first(&avail);
	if (logical_cpu >= nr_cpu_ids)
327
		return 0;
328
	info = kmalloc(sizeof(*info), GFP_KERNEL);
329
330
331
332
333
334
335
336
337
338
339
340
	if (!info)
		return -ENOMEM;
	rc = sclp_get_cpu_info(info);
	if (rc)
		goto out;
	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_id = info->cpu[cpu].address;
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
341
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
342
343
344
345
346
		cpu_set(logical_cpu, cpu_present_map);
		if (cpu >= info->configured)
			smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
		else
			smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
347
348
		logical_cpu = cpumask_next(logical_cpu, &avail);
		if (logical_cpu >= nr_cpu_ids)
349
350
351
			break;
	}
out:
352
	kfree(info);
353
354
355
	return rc;
}

356
static int __smp_rescan_cpus(void)
357
358
359
{
	cpumask_t avail;

360
	cpus_xor(avail, cpu_possible_map, cpu_present_map);
361
362
363
364
	if (smp_use_sigp_detection)
		return smp_rescan_cpus_sigp(avail);
	else
		return smp_rescan_cpus_sclp(avail);
Linus Torvalds's avatar
Linus Torvalds committed
365
366
}

367
368
369
370
371
372
373
374
static void __init smp_detect_cpus(void)
{
	unsigned int cpu, c_cpus, s_cpus;
	struct sclp_cpu_info *info;
	u16 boot_cpu_addr, cpu_addr;

	c_cpus = 1;
	s_cpus = 0;
375
	boot_cpu_addr = __cpu_logical_map[0];
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
	info = kmalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		panic("smp_detect_cpus failed to allocate memory\n");
	/* Use sigp detection algorithm if sclp doesn't work. */
	if (sclp_get_cpu_info(info)) {
		smp_use_sigp_detection = 1;
		for (cpu = 0; cpu <= 65535; cpu++) {
			if (cpu == boot_cpu_addr)
				continue;
			__cpu_logical_map[CPU_INIT_NO] = cpu;
			if (!cpu_stopped(CPU_INIT_NO))
				continue;
			smp_get_save_area(c_cpus, cpu);
			c_cpus++;
		}
		goto out;
	}

	if (info->has_cpu_type) {
		for (cpu = 0; cpu < info->combined; cpu++) {
			if (info->cpu[cpu].address == boot_cpu_addr) {
				smp_cpu_type = info->cpu[cpu].type;
				break;
			}
		}
	}

	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_addr = info->cpu[cpu].address;
		if (cpu_addr == boot_cpu_addr)
			continue;
		__cpu_logical_map[CPU_INIT_NO] = cpu_addr;
		if (!cpu_stopped(CPU_INIT_NO)) {
			s_cpus++;
			continue;
		}
		smp_get_save_area(c_cpus, cpu_addr);
		c_cpus++;
	}
out:
	kfree(info);
419
	pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
420
	get_online_cpus();
421
	__smp_rescan_cpus();
422
	put_online_cpus();
423
424
}

Linus Torvalds's avatar
Linus Torvalds committed
425
/*
426
 *	Activate a secondary processor.
Linus Torvalds's avatar
Linus Torvalds committed
427
 */
428
int __cpuinit start_secondary(void *cpuvoid)
Linus Torvalds's avatar
Linus Torvalds committed
429
{
430
431
	/* Setup the cpu */
	cpu_init();
432
	preempt_disable();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
433
	/* Enable TOD clock interrupts on the secondary cpu. */
434
	init_cpu_timer();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
435
	/* Enable cpu timer interrupts on the secondary cpu. */
436
	init_cpu_vtimer();
Linus Torvalds's avatar
Linus Torvalds committed
437
	/* Enable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
438
439
	pfault_init();

440
441
	/* call cpu notifiers */
	notify_cpu_starting(smp_processor_id());
Linus Torvalds's avatar
Linus Torvalds committed
442
	/* Mark this cpu as online */
443
	ipi_call_lock();
Linus Torvalds's avatar
Linus Torvalds committed
444
	cpu_set(smp_processor_id(), cpu_online_map);
445
	ipi_call_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
446
447
	/* Switch on interrupts */
	local_irq_enable();
448
	/* Print info about this processor */
449
	print_cpu_info();
450
451
452
	/* cpu_idle will call schedule for us */
	cpu_idle();
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
}

static void __init smp_create_idle(unsigned int cpu)
{
	struct task_struct *p;

	/*
	 *  don't care about the psw and regs settings since we'll never
	 *  reschedule the forked task.
	 */
	p = fork_idle(cpu);
	if (IS_ERR(p))
		panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
	current_set[cpu] = p;
}

469
470
471
472
473
474
475
476
477
478
479
480
static int __cpuinit smp_alloc_lowcore(int cpu)
{
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	if (!lowcore)
		return -ENOMEM;
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
	panic_stack = __get_free_page(GFP_KERNEL);
481
482
	if (!panic_stack || !async_stack)
		goto out;
483
484
	memcpy(lowcore, &S390_lowcore, 512);
	memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
485
486
487
488
489
490
491
492
493
	lowcore->async_stack = async_stack + ASYNC_SIZE;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;

#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		unsigned long save_area;

		save_area = get_zeroed_page(GFP_KERNEL);
		if (!save_area)
494
			goto out;
495
496
		lowcore->extended_save_area_addr = (u32) save_area;
	}
497
498
499
#else
	if (vdso_alloc_per_cpu(cpu, lowcore))
		goto out;
500
501
502
503
#endif
	lowcore_ptr[cpu] = lowcore;
	return 0;

504
out:
505
	free_page(panic_stack);
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
	free_pages(async_stack, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	return -ENOMEM;
}

static void smp_free_lowcore(int cpu)
{
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = lowcore_ptr[cpu];
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		free_page((unsigned long) lowcore->extended_save_area_addr);
521
522
#else
	vdso_free_per_cpu(cpu, lowcore);
523
524
525
526
527
528
529
#endif
	free_page(lowcore->panic_stack - PAGE_SIZE);
	free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	lowcore_ptr[cpu] = NULL;
}

Linus Torvalds's avatar
Linus Torvalds committed
530
/* Upping and downing of CPUs */
531
int __cpuinit __cpu_up(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
532
533
{
	struct task_struct *idle;
534
	struct _lowcore *cpu_lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
535
	struct stack_frame *sf;
536
	sigp_ccode ccode;
537
	u32 lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
538

539
540
	if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
		return -EIO;
541
542
	if (smp_alloc_lowcore(cpu))
		return -ENOMEM;
543
544
545
546
547
548
549
550
551
552
553
	do {
		ccode = signal_processor(cpu, sigp_initial_cpu_reset);
		if (ccode == sigp_busy)
			udelay(10);
		if (ccode == sigp_not_operational)
			goto err_out;
	} while (ccode == sigp_busy);

	lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
	while (signal_processor_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
554
555

	idle = current_set[cpu];
556
	cpu_lowcore = lowcore_ptr[cpu];
Linus Torvalds's avatar
Linus Torvalds committed
557
	cpu_lowcore->kernel_stack = (unsigned long)
558
		task_stack_page(idle) + THREAD_SIZE;
559
	cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
Linus Torvalds's avatar
Linus Torvalds committed
560
561
562
563
564
565
	sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
				     - sizeof(struct pt_regs)
				     - sizeof(struct stack_frame));
	memset(sf, 0, sizeof(struct stack_frame));
	sf->gprs[9] = (unsigned long) sf;
	cpu_lowcore->save_area[15] = (unsigned long) sf;
566
	__ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
567
568
569
	asm volatile(
		"	stam	0,15,0(%0)"
		: : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
Linus Torvalds's avatar
Linus Torvalds committed
570
	cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
571
	cpu_lowcore->current_task = (unsigned long) idle;
572
	cpu_lowcore->cpu_nr = cpu;
573
	cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
574
	cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
Linus Torvalds's avatar
Linus Torvalds committed
575
	eieio();
Michael Ryan's avatar
Michael Ryan committed
576

577
	while (signal_processor(cpu, sigp_restart) == sigp_busy)
Michael Ryan's avatar
Michael Ryan committed
578
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
579
580
581
582

	while (!cpu_online(cpu))
		cpu_relax();
	return 0;
583
584
585
586

err_out:
	smp_free_lowcore(cpu);
	return -EIO;
Linus Torvalds's avatar
Linus Torvalds committed
587
588
}

589
static int __init setup_possible_cpus(char *s)
590
{
591
	int pcpus, cpu;
592

593
	pcpus = simple_strtoul(s, NULL, 0);
594
595
	for (cpu = 0; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
		set_cpu_possible(cpu, true);
596
597
598
599
	return 0;
}
early_param("possible_cpus", setup_possible_cpus);

600
601
#ifdef CONFIG_HOTPLUG_CPU

602
int __cpu_disable(void)
Linus Torvalds's avatar
Linus Torvalds committed
603
{
604
	struct ec_creg_mask_parms cr_parms;
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
605
	int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
606

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
607
	cpu_clear(cpu, cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
608
609

	/* Disable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
610
	pfault_fini();
Linus Torvalds's avatar
Linus Torvalds committed
611

612
613
	memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
	memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
614

615
	/* disable all external interrupts */
Linus Torvalds's avatar
Linus Torvalds committed
616
	cr_parms.orvals[0] = 0;
617
618
	cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
				1 << 11 | 1 << 10 | 1 <<  6 | 1 <<  4);
Linus Torvalds's avatar
Linus Torvalds committed
619
620
	/* disable all I/O interrupts */
	cr_parms.orvals[6] = 0;
621
622
	cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
				1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
Linus Torvalds's avatar
Linus Torvalds committed
623
624
	/* disable most machine checks */
	cr_parms.orvals[14] = 0;
625
626
	cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
				 1 << 25 | 1 << 24);
627

Linus Torvalds's avatar
Linus Torvalds committed
628
629
630
631
632
	smp_ctl_bit_callback(&cr_parms);

	return 0;
}

633
void __cpu_die(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
634
635
636
637
{
	/* Wait until target cpu is down */
	while (!smp_cpu_not_running(cpu))
		cpu_relax();
638
	smp_free_lowcore(cpu);
639
	pr_info("Processor %d stopped\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
640
641
}

642
void cpu_die(void)
Linus Torvalds's avatar
Linus Torvalds committed
643
644
645
646
{
	idle_task_exit();
	signal_processor(smp_processor_id(), sigp_stop);
	BUG();
647
	for (;;);
Linus Torvalds's avatar
Linus Torvalds committed
648
649
}

650
651
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
652
653
void __init smp_prepare_cpus(unsigned int max_cpus)
{
654
655
656
657
658
#ifndef CONFIG_64BIT
	unsigned long save_area = 0;
#endif
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
659
	unsigned int cpu;
660
	int lc_order;
661

662
663
	smp_detect_cpus();

664
665
666
	/* request the 0x1201 emergency signal external interrupt */
	if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1201");
667
	print_cpu_info();
Linus Torvalds's avatar
Linus Torvalds committed
668

669
670
671
672
673
	/* Reallocate current lowcore, but keep its contents. */
	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	panic_stack = __get_free_page(GFP_KERNEL);
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
674
	BUG_ON(!lowcore || !panic_stack || !async_stack);
675
#ifndef CONFIG_64BIT
676
	if (MACHINE_HAS_IEEE)
677
		save_area = get_zeroed_page(GFP_KERNEL);
678
#endif
679
680
681
682
683
684
685
686
687
	local_irq_disable();
	local_mcck_disable();
	lowcore_ptr[smp_processor_id()] = lowcore;
	*lowcore = S390_lowcore;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;
	lowcore->async_stack = async_stack + ASYNC_SIZE;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		lowcore->extended_save_area_addr = (u32) save_area;
688
#else
Heiko Carstens's avatar
Heiko Carstens committed
689
690
	if (vdso_alloc_per_cpu(smp_processor_id(), lowcore))
		BUG();
691
692
693
694
#endif
	set_prefix((u32)(unsigned long) lowcore);
	local_mcck_enable();
	local_irq_enable();
695
	for_each_possible_cpu(cpu)
Linus Torvalds's avatar
Linus Torvalds committed
696
697
698
699
		if (cpu != smp_processor_id())
			smp_create_idle(cpu);
}

700
void __init smp_prepare_boot_cpu(void)
Linus Torvalds's avatar
Linus Torvalds committed
701
702
703
{
	BUG_ON(smp_processor_id() != 0);

704
705
	current_thread_info()->cpu = 0;
	cpu_set(0, cpu_present_map);
Linus Torvalds's avatar
Linus Torvalds committed
706
707
708
	cpu_set(0, cpu_online_map);
	S390_lowcore.percpu_offset = __per_cpu_offset[0];
	current_set[0] = current;
709
	smp_cpu_state[0] = CPU_STATE_CONFIGURED;
710
	smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
Linus Torvalds's avatar
Linus Torvalds committed
711
712
}

713
void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds's avatar
Linus Torvalds committed
714
715
716
717
718
719
720
721
722
723
724
{
}

/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
725
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
726
727
}

728
#ifdef CONFIG_HOTPLUG_CPU
729
730
static ssize_t cpu_configure_show(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
731
732
733
734
735
736
737
738
739
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

740
741
742
static ssize_t cpu_configure_store(struct sys_device *dev,
				  struct sysdev_attribute *attr,
				  const char *buf, size_t count)
743
744
745
746
747
748
749
750
751
752
{
	int cpu = dev->id;
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;

753
	get_online_cpus();
754
	mutex_lock(&smp_cpu_state_mutex);
755
756
757
758
759
760
761
762
	rc = -EBUSY;
	if (cpu_online(cpu))
		goto out;
	rc = 0;
	switch (val) {
	case 0:
		if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
			rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
763
			if (!rc) {
764
				smp_cpu_state[cpu] = CPU_STATE_STANDBY;
765
766
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
767
768
769
770
771
		}
		break;
	case 1:
		if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
			rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
772
			if (!rc) {
773
				smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
774
775
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
776
777
778
779
780
781
782
		}
		break;
	default:
		break;
	}
out:
	mutex_unlock(&smp_cpu_state_mutex);
783
	put_online_cpus();
784
785
786
787
788
	return rc ? rc : count;
}
static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
#endif /* CONFIG_HOTPLUG_CPU */

789
790
static ssize_t cpu_polarization_show(struct sys_device *dev,
				     struct sysdev_attribute *attr, char *buf)
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	switch (smp_cpu_polarization[cpu]) {
	case POLARIZATION_HRZ:
		count = sprintf(buf, "horizontal\n");
		break;
	case POLARIZATION_VL:
		count = sprintf(buf, "vertical:low\n");
		break;
	case POLARIZATION_VM:
		count = sprintf(buf, "vertical:medium\n");
		break;
	case POLARIZATION_VH:
		count = sprintf(buf, "vertical:high\n");
		break;
	default:
		count = sprintf(buf, "unknown\n");
		break;
	}
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);

818
819
static ssize_t show_cpu_address(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
820
821
822
823
824
825
826
827
828
829
830
{
	return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
}
static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);


static struct attribute *cpu_common_attrs[] = {
#ifdef CONFIG_HOTPLUG_CPU
	&attr_configure.attr,
#endif
	&attr_address.attr,
831
	&attr_polarization.attr,
832
833
834
835
836
837
	NULL,
};

static struct attribute_group cpu_common_attr_group = {
	.attrs = cpu_common_attrs,
};
Linus Torvalds's avatar
Linus Torvalds committed
838

839
840
static ssize_t show_capability(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
841
842
843
844
845
846
847
848
849
850
851
{
	unsigned int capability;
	int rc;

	rc = get_cpu_capability(&capability);
	if (rc)
		return rc;
	return sprintf(buf, "%u\n", capability);
}
static SYSDEV_ATTR(capability, 0444, show_capability, NULL);

852
853
static ssize_t show_idle_count(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
854
855
856
857
858
{
	struct s390_idle_data *idle;
	unsigned long long idle_count;

	idle = &per_cpu(s390_idle, dev->id);
859
	spin_lock(&idle->lock);
860
	idle_count = idle->idle_count;
861
862
863
	if (idle->idle_enter)
		idle_count++;
	spin_unlock(&idle->lock);
864
865
866
867
	return sprintf(buf, "%llu\n", idle_count);
}
static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);

868
869
static ssize_t show_idle_time(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
870
871
{
	struct s390_idle_data *idle;
872
	unsigned long long now, idle_time, idle_enter;
873
874

	idle = &per_cpu(s390_idle, dev->id);
875
876
877
878
879
880
881
882
	spin_lock(&idle->lock);
	now = get_clock();
	idle_time = idle->idle_time;
	idle_enter = idle->idle_enter;
	if (idle_enter != 0ULL && idle_enter < now)
		idle_time += now - idle_enter;
	spin_unlock(&idle->lock);
	return sprintf(buf, "%llu\n", idle_time >> 12);
883
}
884
static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
885

886
static struct attribute *cpu_online_attrs[] = {
887
888
	&attr_capability.attr,
	&attr_idle_count.attr,
889
	&attr_idle_time_us.attr,
890
891
892
	NULL,
};

893
894
static struct attribute_group cpu_online_attr_group = {
	.attrs = cpu_online_attrs,
895
896
};

897
898
899
900
901
902
static int __cpuinit smp_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned int)(long)hcpu;
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
903
	struct s390_idle_data *idle;
904
905
906

	switch (action) {
	case CPU_ONLINE:
907
	case CPU_ONLINE_FROZEN:
908
909
910
911
912
913
		idle = &per_cpu(s390_idle, cpu);
		spin_lock_irq(&idle->lock);
		idle->idle_enter = 0;
		idle->idle_time = 0;
		idle->idle_count = 0;
		spin_unlock_irq(&idle->lock);
914
		if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
915
916
917
			return NOTIFY_BAD;
		break;
	case CPU_DEAD:
918
	case CPU_DEAD_FROZEN:
919
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
920
921
922
923
924
925
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata smp_cpu_nb = {
926
	.notifier_call = smp_cpu_notify,
927
928
};

929
static int __devinit smp_add_present_cpu(int cpu)
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
{
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
	int rc;

	c->hotpluggable = 1;
	rc = register_cpu(c, cpu);
	if (rc)
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
	if (rc)
		goto out_cpu;
	if (!cpu_online(cpu))
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
	if (!rc)
		return 0;
	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
out_cpu:
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu(c);
#endif
out:
	return rc;
}

#ifdef CONFIG_HOTPLUG_CPU
957

958
int __ref smp_rescan_cpus(void)
959
960
961
962
963
{
	cpumask_t newcpus;
	int cpu;
	int rc;

964
	get_online_cpus();
965
	mutex_lock(&smp_cpu_state_mutex);
966
	newcpus = cpu_present_map;
967
	rc = __smp_rescan_cpus();
968
969
970
971
972
973
974
975
976
977
978
	if (rc)
		goto out;
	cpus_andnot(newcpus, cpu_present_map, newcpus);
	for_each_cpu_mask(cpu, newcpus) {
		rc = smp_add_present_cpu(cpu);
		if (rc)
			cpu_clear(cpu, cpu_present_map);
	}
	rc = 0;
out:
	mutex_unlock(&smp_cpu_state_mutex);
979
	put_online_cpus();
980
981
	if (!cpus_empty(newcpus))
		topology_schedule_update();
982
983
984
	return rc;
}

985
static ssize_t __ref rescan_store(struct sysdev_class *class, const char *buf,
986
987
988
989
990
				  size_t count)
{
	int rc;

	rc = smp_rescan_cpus();
991
992
	return rc ? rc : count;
}
993
static SYSDEV_CLASS_ATTR(rescan, 0200, NULL, rescan_store);
994
995
#endif /* CONFIG_HOTPLUG_CPU */

996
static ssize_t dispatching_show(struct sysdev_class *class, char *buf)
997
998
999
1000
1001
1002
1003
1004
1005
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", cpu_management);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

1006
1007
static ssize_t dispatching_store(struct sysdev_class *dev, const char *buf,
				 size_t count)
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
{
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
	rc = 0;
	get_online_cpus();
1018
	mutex_lock(&smp_cpu_state_mutex);
1019
1020
1021
1022
1023
1024
1025
	if (cpu_management == val)
		goto out;
	rc = topology_set_cpu_management(val);
	if (!rc)
		cpu_management = val;
out:
	mutex_unlock(&smp_cpu_state_mutex);
1026
	put_online_cpus();
1027
1028
	return rc ? rc : count;
}
1029
1030
static SYSDEV_CLASS_ATTR(dispatching, 0644, dispatching_show,
			 dispatching_store);
1031

Linus Torvalds's avatar
Linus Torvalds committed
1032
1033
1034
static int __init topology_init(void)
{
	int cpu;
1035
	int rc;
1036
1037

	register_cpu_notifier(&smp_cpu_nb);
Linus Torvalds's avatar
Linus Torvalds committed
1038

1039
#ifdef CONFIG_HOTPLUG_CPU
1040
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_rescan);
1041
1042
1043
	if (rc)
		return rc;
#endif
1044
	rc = sysdev_class_create_file(&cpu_sysdev_class, &attr_dispatching);
1045
1046
	if (rc)
		return rc;
1047
1048
	for_each_present_cpu(cpu) {
		rc = smp_add_present_cpu(cpu);
1049
1050
		if (rc)
			return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1051
1052
1053
1054
	}
	return 0;
}
subsys_initcall(topology_init);