smp.c 29.1 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
/*
 *  arch/s390/kernel/smp.c
 *
4
 *    Copyright IBM Corp. 1999,2007
Linus Torvalds's avatar
Linus Torvalds committed
5
 *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6
7
 *		 Martin Schwidefsky (schwidefsky@de.ibm.com)
 *		 Heiko Carstens (heiko.carstens@de.ibm.com)
Linus Torvalds's avatar
Linus Torvalds committed
8
 *
9
 *  based on other smp stuff by
Linus Torvalds's avatar
Linus Torvalds committed
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
 *    (c) 1998 Ingo Molnar
 *
 * We work with logical cpu numbering everywhere we can. The only
 * functions using the real cpu address (got from STAP) are the sigp
 * functions. For all other functions we use the identity mapping.
 * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
 * used e.g. to find the idle task belonging to a logical cpu. Every array
 * in the kernel is sorted by the logical cpu number and not by the physical
 * one which is causing all the confusion with __cpu_logical_map and
 * cpu_number_map in other architectures.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/mm.h>
Alexey Dobriyan's avatar
Alexey Dobriyan committed
26
#include <linux/err.h>
Linus Torvalds's avatar
Linus Torvalds committed
27
28
29
30
31
32
#include <linux/spinlock.h>
#include <linux/kernel_stat.h>
#include <linux/delay.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/cpu.h>
33
#include <linux/timex.h>
Michael Holzheu's avatar
Michael Holzheu committed
34
#include <linux/bootmem.h>
Michael Holzheu's avatar
Michael Holzheu committed
35
#include <asm/ipl.h>
36
#include <asm/setup.h>
Linus Torvalds's avatar
Linus Torvalds committed
37
38
39
40
41
42
#include <asm/sigp.h>
#include <asm/pgalloc.h>
#include <asm/irq.h>
#include <asm/s390_ext.h>
#include <asm/cpcmd.h>
#include <asm/tlbflush.h>
43
#include <asm/timer.h>
Michael Holzheu's avatar
Michael Holzheu committed
44
#include <asm/lowcore.h>
45
#include <asm/sclp.h>
46
#include <asm/cpu.h>
47
#include "entry.h"
Linus Torvalds's avatar
Linus Torvalds committed
48
49
50
51
52

/*
 * An array with a pointer the lowcore of every CPU.
 */
struct _lowcore *lowcore_ptr[NR_CPUS];
53
EXPORT_SYMBOL(lowcore_ptr);
Linus Torvalds's avatar
Linus Torvalds committed
54

55
cpumask_t cpu_online_map = CPU_MASK_NONE;
56
57
EXPORT_SYMBOL(cpu_online_map);

58
cpumask_t cpu_possible_map = CPU_MASK_ALL;
59
EXPORT_SYMBOL(cpu_possible_map);
Linus Torvalds's avatar
Linus Torvalds committed
60
61
62

static struct task_struct *current_set[NR_CPUS];

63
64
65
66
67
68
69
70
static u8 smp_cpu_type;
static int smp_use_sigp_detection;

enum s390_cpu_state {
	CPU_STATE_STANDBY,
	CPU_STATE_CONFIGURED,
};

71
DEFINE_MUTEX(smp_cpu_state_mutex);
72
int smp_cpu_polarization[NR_CPUS];
73
static int smp_cpu_state[NR_CPUS];
74
static int cpu_management;
75
76
77

static DEFINE_PER_CPU(struct cpu, cpu_devices);

Linus Torvalds's avatar
Linus Torvalds committed
78
79
80
static void smp_ext_bitcall(int, ec_bit_sig);

/*
Jan Glauber's avatar
Jan Glauber committed
81
82
 * Structure and data for __smp_call_function_map(). This is designed to
 * minimise static memory requirements. It also looks cleaner.
Linus Torvalds's avatar
Linus Torvalds committed
83
84
85
86
87
88
 */
static DEFINE_SPINLOCK(call_lock);

struct call_data_struct {
	void (*func) (void *info);
	void *info;
Jan Glauber's avatar
Jan Glauber committed
89
90
	cpumask_t started;
	cpumask_t finished;
Linus Torvalds's avatar
Linus Torvalds committed
91
92
93
	int wait;
};

94
static struct call_data_struct *call_data;
Linus Torvalds's avatar
Linus Torvalds committed
95
96
97
98
99
100
101
102
103
104

/*
 * 'Call function' interrupt callback
 */
static void do_call_function(void)
{
	void (*func) (void *info) = call_data->func;
	void *info = call_data->info;
	int wait = call_data->wait;

Jan Glauber's avatar
Jan Glauber committed
105
	cpu_set(smp_processor_id(), call_data->started);
Linus Torvalds's avatar
Linus Torvalds committed
106
107
	(*func)(info);
	if (wait)
Jan Glauber's avatar
Jan Glauber committed
108
		cpu_set(smp_processor_id(), call_data->finished);;
Linus Torvalds's avatar
Linus Torvalds committed
109
110
}

Jan Glauber's avatar
Jan Glauber committed
111
static void __smp_call_function_map(void (*func) (void *info), void *info,
112
				    int wait, cpumask_t map)
Linus Torvalds's avatar
Linus Torvalds committed
113
114
{
	struct call_data_struct data;
Jan Glauber's avatar
Jan Glauber committed
115
	int cpu, local = 0;
Linus Torvalds's avatar
Linus Torvalds committed
116

Jan Glauber's avatar
Jan Glauber committed
117
	/*
118
	 * Can deadlock when interrupts are disabled or if in wrong context.
Jan Glauber's avatar
Jan Glauber committed
119
	 */
120
	WARN_ON(irqs_disabled() || in_irq());
Linus Torvalds's avatar
Linus Torvalds committed
121

Jan Glauber's avatar
Jan Glauber committed
122
123
124
125
126
127
128
129
130
131
132
133
	/*
	 * Check for local function call. We have to have the same call order
	 * as in on_each_cpu() because of machine_restart_smp().
	 */
	if (cpu_isset(smp_processor_id(), map)) {
		local = 1;
		cpu_clear(smp_processor_id(), map);
	}

	cpus_and(map, map, cpu_online_map);
	if (cpus_empty(map))
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
134
135
136

	data.func = func;
	data.info = info;
Jan Glauber's avatar
Jan Glauber committed
137
	data.started = CPU_MASK_NONE;
Linus Torvalds's avatar
Linus Torvalds committed
138
139
	data.wait = wait;
	if (wait)
Jan Glauber's avatar
Jan Glauber committed
140
		data.finished = CPU_MASK_NONE;
Linus Torvalds's avatar
Linus Torvalds committed
141
142

	call_data = &data;
Jan Glauber's avatar
Jan Glauber committed
143
144
145

	for_each_cpu_mask(cpu, map)
		smp_ext_bitcall(cpu, ec_call_function);
Linus Torvalds's avatar
Linus Torvalds committed
146
147

	/* Wait for response */
Jan Glauber's avatar
Jan Glauber committed
148
	while (!cpus_equal(map, data.started))
Linus Torvalds's avatar
Linus Torvalds committed
149
150
		cpu_relax();
	if (wait)
Jan Glauber's avatar
Jan Glauber committed
151
		while (!cpus_equal(map, data.finished))
Linus Torvalds's avatar
Linus Torvalds committed
152
			cpu_relax();
Jan Glauber's avatar
Jan Glauber committed
153
out:
154
155
	if (local) {
		local_irq_disable();
Jan Glauber's avatar
Jan Glauber committed
156
		func(info);
157
158
		local_irq_enable();
	}
Linus Torvalds's avatar
Linus Torvalds committed
159
160
161
}

/*
Jan Glauber's avatar
Jan Glauber committed
162
163
164
165
 * smp_call_function:
 * @func: the function to run; this must be fast and non-blocking
 * @info: an arbitrary pointer to pass to the function
 * @wait: if true, wait (atomically) until function has completed on other CPUs
Linus Torvalds's avatar
Linus Torvalds committed
166
 *
Jan Glauber's avatar
Jan Glauber committed
167
 * Run a function on all other CPUs.
Linus Torvalds's avatar
Linus Torvalds committed
168
 *
169
170
 * You must not call this function with disabled interrupts, from a
 * hardware interrupt handler or from a bottom half.
Linus Torvalds's avatar
Linus Torvalds committed
171
 */
172
int smp_call_function(void (*func) (void *info), void *info, int wait)
Linus Torvalds's avatar
Linus Torvalds committed
173
{
Jan Glauber's avatar
Jan Glauber committed
174
	cpumask_t map;
Linus Torvalds's avatar
Linus Torvalds committed
175

176
	spin_lock(&call_lock);
Jan Glauber's avatar
Jan Glauber committed
177
178
	map = cpu_online_map;
	cpu_clear(smp_processor_id(), map);
179
	__smp_call_function_map(func, info, wait, map);
180
	spin_unlock(&call_lock);
Jan Glauber's avatar
Jan Glauber committed
181
182
183
	return 0;
}
EXPORT_SYMBOL(smp_call_function);
Linus Torvalds's avatar
Linus Torvalds committed
184

Jan Glauber's avatar
Jan Glauber committed
185
/*
186
187
 * smp_call_function_single:
 * @cpu: the CPU where func should run
Jan Glauber's avatar
Jan Glauber committed
188
189
190
191
192
193
 * @func: the function to run; this must be fast and non-blocking
 * @info: an arbitrary pointer to pass to the function
 * @wait: if true, wait (atomically) until function has completed on other CPUs
 *
 * Run a function on one processor.
 *
194
195
 * You must not call this function with disabled interrupts, from a
 * hardware interrupt handler or from a bottom half.
Jan Glauber's avatar
Jan Glauber committed
196
 */
197
int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
198
			     int wait)
Jan Glauber's avatar
Jan Glauber committed
199
{
200
	spin_lock(&call_lock);
201
	__smp_call_function_map(func, info, wait, cpumask_of_cpu(cpu));
202
	spin_unlock(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
203
204
	return 0;
}
205
EXPORT_SYMBOL(smp_call_function_single);
Linus Torvalds's avatar
Linus Torvalds committed
206

207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/**
 * smp_call_function_mask(): Run a function on a set of other CPUs.
 * @mask: The set of cpus to run on.  Must not include the current cpu.
 * @func: The function to run. This must be fast and non-blocking.
 * @info: An arbitrary pointer to pass to the function.
 * @wait: If true, wait (atomically) until function has completed on other CPUs.
 *
 * Returns 0 on success, else a negative status code.
 *
 * If @wait is true, then returns once @func has returned; otherwise
 * it returns just before the target cpu calls @func.
 *
 * You must not call this function with disabled interrupts or from a
 * hardware interrupt handler or from a bottom half handler.
 */
222
223
int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info,
			   int wait)
224
{
225
	spin_lock(&call_lock);
226
	cpu_clear(smp_processor_id(), mask);
227
	__smp_call_function_map(func, info, wait, mask);
228
	spin_unlock(&call_lock);
229
230
231
232
	return 0;
}
EXPORT_SYMBOL(smp_call_function_mask);

233
void smp_send_stop(void)
Linus Torvalds's avatar
Linus Torvalds committed
234
{
235
	int cpu, rc;
Linus Torvalds's avatar
Linus Torvalds committed
236

237
238
	/* Disable all interrupts/machine checks */
	__load_psw_mask(psw_kernel_bits & ~PSW_MASK_MCHECK);
Linus Torvalds's avatar
Linus Torvalds committed
239

240
241
	/* write magic number to zero page (absolute 0) */
	lowcore_ptr[smp_processor_id()]->panic_magic = __PANIC_MAGIC;
Linus Torvalds's avatar
Linus Torvalds committed
242

243
	/* stop all processors */
Linus Torvalds's avatar
Linus Torvalds committed
244
245
246
247
	for_each_online_cpu(cpu) {
		if (cpu == smp_processor_id())
			continue;
		do {
248
			rc = signal_processor(cpu, sigp_stop);
249
		} while (rc == sigp_busy);
Linus Torvalds's avatar
Linus Torvalds committed
250

251
		while (!smp_cpu_not_running(cpu))
Heiko Carstens's avatar
Heiko Carstens committed
252
253
254
255
			cpu_relax();
	}
}

Linus Torvalds's avatar
Linus Torvalds committed
256
257
258
259
260
/*
 * This is the main routine where commands issued by other
 * cpus are handled.
 */

261
static void do_ext_call_interrupt(__u16 code)
Linus Torvalds's avatar
Linus Torvalds committed
262
{
263
	unsigned long bits;
Linus Torvalds's avatar
Linus Torvalds committed
264

265
266
267
268
269
270
	/*
	 * handle bit signal external calls
	 *
	 * For the ec_schedule signal we have to do nothing. All the work
	 * is done automatically when we return from the interrupt.
	 */
Linus Torvalds's avatar
Linus Torvalds committed
271
272
	bits = xchg(&S390_lowcore.ext_call_fast, 0);

273
	if (test_bit(ec_call_function, &bits))
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278
279
280
281
282
		do_call_function();
}

/*
 * Send an external call sigp to another cpu and return without waiting
 * for its completion.
 */
static void smp_ext_bitcall(int cpu, ec_bit_sig sig)
{
283
284
285
	/*
	 * Set signaling bit in lowcore of target cpu and kick it
	 */
Linus Torvalds's avatar
Linus Torvalds committed
286
	set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
287
	while (signal_processor(cpu, sigp_emergency_signal) == sigp_busy)
Linus Torvalds's avatar
Linus Torvalds committed
288
289
290
		udelay(10);
}

291
#ifndef CONFIG_64BIT
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
/*
 * this function sends a 'purge tlb' signal to another CPU.
 */
295
static void smp_ptlb_callback(void *info)
Linus Torvalds's avatar
Linus Torvalds committed
296
{
Martin Schwidefsky's avatar
Martin Schwidefsky committed
297
	__tlb_flush_local();
Linus Torvalds's avatar
Linus Torvalds committed
298
299
300
301
}

void smp_ptlb_all(void)
{
302
	on_each_cpu(smp_ptlb_callback, NULL, 1);
Linus Torvalds's avatar
Linus Torvalds committed
303
304
}
EXPORT_SYMBOL(smp_ptlb_all);
305
#endif /* ! CONFIG_64BIT */
Linus Torvalds's avatar
Linus Torvalds committed
306
307
308
309
310
311
312
313

/*
 * this function sends a 'reschedule' IPI to another CPU.
 * it goes straight through and wastes no time serializing
 * anything. Worst case is that we lose a reschedule ...
 */
void smp_send_reschedule(int cpu)
{
314
	smp_ext_bitcall(cpu, ec_schedule);
Linus Torvalds's avatar
Linus Torvalds committed
315
316
317
318
319
}

/*
 * parameter area for the set/clear control bit callbacks
 */
320
struct ec_creg_mask_parms {
Linus Torvalds's avatar
Linus Torvalds committed
321
322
	unsigned long orvals[16];
	unsigned long andvals[16];
323
};
Linus Torvalds's avatar
Linus Torvalds committed
324
325
326
327

/*
 * callback for setting/clearing control bits
 */
328
329
static void smp_ctl_bit_callback(void *info)
{
330
	struct ec_creg_mask_parms *pp = info;
Linus Torvalds's avatar
Linus Torvalds committed
331
332
	unsigned long cregs[16];
	int i;
333

334
335
	__ctl_store(cregs, 0, 15);
	for (i = 0; i <= 15; i++)
Linus Torvalds's avatar
Linus Torvalds committed
336
		cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
337
	__ctl_load(cregs, 0, 15);
Linus Torvalds's avatar
Linus Torvalds committed
338
339
340
341
342
}

/*
 * Set a bit in a control register of all cpus
 */
343
344
345
void smp_ctl_set_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
346

347
348
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
349
	parms.orvals[cr] = 1 << bit;
350
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
351
}
352
EXPORT_SYMBOL(smp_ctl_set_bit);
Linus Torvalds's avatar
Linus Torvalds committed
353
354
355
356

/*
 * Clear a bit in a control register of all cpus
 */
357
358
359
void smp_ctl_clear_bit(int cr, int bit)
{
	struct ec_creg_mask_parms parms;
Linus Torvalds's avatar
Linus Torvalds committed
360

361
362
	memset(&parms.orvals, 0, sizeof(parms.orvals));
	memset(&parms.andvals, 0xff, sizeof(parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
363
	parms.andvals[cr] = ~(1L << bit);
364
	on_each_cpu(smp_ctl_bit_callback, &parms, 1);
Linus Torvalds's avatar
Linus Torvalds committed
365
}
366
EXPORT_SYMBOL(smp_ctl_clear_bit);
Linus Torvalds's avatar
Linus Torvalds committed
367

368
369
370
371
372
373
374
/*
 * In early ipl state a temp. logically cpu number is needed, so the sigp
 * functions can be used to sense other cpus. Since NR_CPUS is >= 2 on
 * CONFIG_SMP and the ipl cpu is logical cpu 0, it must be 1.
 */
#define CPU_INIT_NO	1

Michael Holzheu's avatar
Michael Holzheu committed
375
376
377
378
379
380
381
382
383
384
385
#if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_ZFCPDUMP_MODULE)

/*
 * zfcpdump_prefix_array holds prefix registers for the following scenario:
 * 64 bit zfcpdump kernel and 31 bit kernel which is to be dumped. We have to
 * save its prefix registers, since they get lost, when switching from 31 bit
 * to 64 bit.
 */
unsigned int zfcpdump_prefix_array[NR_CPUS + 1] \
	__attribute__((__section__(".data")));

386
static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
Michael Holzheu's avatar
Michael Holzheu committed
387
388
389
{
	if (ipl_info.type != IPL_TYPE_FCP_DUMP)
		return;
390
391
392
393
	if (cpu >= NR_CPUS) {
		printk(KERN_WARNING "Registers for cpu %i not saved since dump "
		       "kernel was compiled with NR_CPUS=%i\n", cpu, NR_CPUS);
		return;
Michael Holzheu's avatar
Michael Holzheu committed
394
	}
395
	zfcpdump_save_areas[cpu] = kmalloc(sizeof(union save_area), GFP_KERNEL);
396
397
398
	__cpu_logical_map[CPU_INIT_NO] = (__u16) phy_cpu;
	while (signal_processor(CPU_INIT_NO, sigp_stop_and_store_status) ==
	       sigp_busy)
399
400
401
402
403
404
405
406
		cpu_relax();
	memcpy(zfcpdump_save_areas[cpu],
	       (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
	       SAVE_AREA_SIZE);
#ifdef CONFIG_64BIT
	/* copy original prefix register */
	zfcpdump_save_areas[cpu]->s390x.pref_reg = zfcpdump_prefix_array[cpu];
#endif
Michael Holzheu's avatar
Michael Holzheu committed
407
408
409
410
411
412
}

union save_area *zfcpdump_save_areas[NR_CPUS + 1];
EXPORT_SYMBOL_GPL(zfcpdump_save_areas);

#else
413
414
415
416

static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }

#endif /* CONFIG_ZFCPDUMP || CONFIG_ZFCPDUMP_MODULE */
Michael Holzheu's avatar
Michael Holzheu committed
417

418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
static int cpu_stopped(int cpu)
{
	__u32 status;

	/* Check for stopped state */
	if (signal_processor_ps(&status, 0, cpu, sigp_sense) ==
	    sigp_status_stored) {
		if (status & 0x40)
			return 1;
	}
	return 0;
}

static int cpu_known(int cpu_id)
{
	int cpu;

	for_each_present_cpu(cpu) {
		if (__cpu_logical_map[cpu] == cpu_id)
			return 1;
	}
	return 0;
}

static int smp_rescan_cpus_sigp(cpumask_t avail)
{
	int cpu_id, logical_cpu;

	logical_cpu = first_cpu(avail);
	if (logical_cpu == NR_CPUS)
		return 0;
	for (cpu_id = 0; cpu_id <= 65535; cpu_id++) {
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
453
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
454
455
456
457
458
459
460
461
462
463
464
		if (!cpu_stopped(logical_cpu))
			continue;
		cpu_set(logical_cpu, cpu_present_map);
		smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
		logical_cpu = next_cpu(logical_cpu, avail);
		if (logical_cpu == NR_CPUS)
			break;
	}
	return 0;
}

465
static int smp_rescan_cpus_sclp(cpumask_t avail)
466
467
468
469
470
471
472
473
{
	struct sclp_cpu_info *info;
	int cpu_id, logical_cpu, cpu;
	int rc;

	logical_cpu = first_cpu(avail);
	if (logical_cpu == NR_CPUS)
		return 0;
474
	info = kmalloc(sizeof(*info), GFP_KERNEL);
475
476
477
478
479
480
481
482
483
484
485
486
	if (!info)
		return -ENOMEM;
	rc = sclp_get_cpu_info(info);
	if (rc)
		goto out;
	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_id = info->cpu[cpu].address;
		if (cpu_known(cpu_id))
			continue;
		__cpu_logical_map[logical_cpu] = cpu_id;
487
		smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
488
489
490
491
492
493
494
495
496
497
		cpu_set(logical_cpu, cpu_present_map);
		if (cpu >= info->configured)
			smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
		else
			smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
		logical_cpu = next_cpu(logical_cpu, avail);
		if (logical_cpu == NR_CPUS)
			break;
	}
out:
498
	kfree(info);
499
500
501
	return rc;
}

502
static int __smp_rescan_cpus(void)
503
504
505
{
	cpumask_t avail;

506
	cpus_xor(avail, cpu_possible_map, cpu_present_map);
507
508
509
510
	if (smp_use_sigp_detection)
		return smp_rescan_cpus_sigp(avail);
	else
		return smp_rescan_cpus_sclp(avail);
Linus Torvalds's avatar
Linus Torvalds committed
511
512
}

513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
static void __init smp_detect_cpus(void)
{
	unsigned int cpu, c_cpus, s_cpus;
	struct sclp_cpu_info *info;
	u16 boot_cpu_addr, cpu_addr;

	c_cpus = 1;
	s_cpus = 0;
	boot_cpu_addr = S390_lowcore.cpu_data.cpu_addr;
	info = kmalloc(sizeof(*info), GFP_KERNEL);
	if (!info)
		panic("smp_detect_cpus failed to allocate memory\n");
	/* Use sigp detection algorithm if sclp doesn't work. */
	if (sclp_get_cpu_info(info)) {
		smp_use_sigp_detection = 1;
		for (cpu = 0; cpu <= 65535; cpu++) {
			if (cpu == boot_cpu_addr)
				continue;
			__cpu_logical_map[CPU_INIT_NO] = cpu;
			if (!cpu_stopped(CPU_INIT_NO))
				continue;
			smp_get_save_area(c_cpus, cpu);
			c_cpus++;
		}
		goto out;
	}

	if (info->has_cpu_type) {
		for (cpu = 0; cpu < info->combined; cpu++) {
			if (info->cpu[cpu].address == boot_cpu_addr) {
				smp_cpu_type = info->cpu[cpu].type;
				break;
			}
		}
	}

	for (cpu = 0; cpu < info->combined; cpu++) {
		if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
			continue;
		cpu_addr = info->cpu[cpu].address;
		if (cpu_addr == boot_cpu_addr)
			continue;
		__cpu_logical_map[CPU_INIT_NO] = cpu_addr;
		if (!cpu_stopped(CPU_INIT_NO)) {
			s_cpus++;
			continue;
		}
		smp_get_save_area(c_cpus, cpu_addr);
		c_cpus++;
	}
out:
	kfree(info);
	printk(KERN_INFO "CPUs: %d configured, %d standby\n", c_cpus, s_cpus);
566
	get_online_cpus();
567
	__smp_rescan_cpus();
568
	put_online_cpus();
569
570
}

Linus Torvalds's avatar
Linus Torvalds committed
571
/*
572
 *	Activate a secondary processor.
Linus Torvalds's avatar
Linus Torvalds committed
573
 */
574
int __cpuinit start_secondary(void *cpuvoid)
Linus Torvalds's avatar
Linus Torvalds committed
575
{
576
577
	/* Setup the cpu */
	cpu_init();
578
	preempt_disable();
Martin Schwidefsky's avatar
Martin Schwidefsky committed
579
	/* Enable TOD clock interrupts on the secondary cpu. */
580
	init_cpu_timer();
Linus Torvalds's avatar
Linus Torvalds committed
581
#ifdef CONFIG_VIRT_TIMER
Martin Schwidefsky's avatar
Martin Schwidefsky committed
582
	/* Enable cpu timer interrupts on the secondary cpu. */
583
	init_cpu_vtimer();
Linus Torvalds's avatar
Linus Torvalds committed
584
585
#endif
	/* Enable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
586
587
	pfault_init();

588
589
	/* call cpu notifiers */
	notify_cpu_starting(smp_processor_id());
Linus Torvalds's avatar
Linus Torvalds committed
590
	/* Mark this cpu as online */
591
	spin_lock(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
592
	cpu_set(smp_processor_id(), cpu_online_map);
593
	spin_unlock(&call_lock);
Linus Torvalds's avatar
Linus Torvalds committed
594
595
	/* Switch on interrupts */
	local_irq_enable();
596
597
598
599
600
	/* Print info about this processor */
	print_cpu_info(&S390_lowcore.cpu_data);
	/* cpu_idle will call schedule for us */
	cpu_idle();
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
}

static void __init smp_create_idle(unsigned int cpu)
{
	struct task_struct *p;

	/*
	 *  don't care about the psw and regs settings since we'll never
	 *  reschedule the forked task.
	 */
	p = fork_idle(cpu);
	if (IS_ERR(p))
		panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p));
	current_set[cpu] = p;
}

617
618
619
620
621
622
623
624
625
626
627
628
static int __cpuinit smp_alloc_lowcore(int cpu)
{
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	if (!lowcore)
		return -ENOMEM;
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
	panic_stack = __get_free_page(GFP_KERNEL);
629
630
	if (!panic_stack || !async_stack)
		goto out;
631
632
	memcpy(lowcore, &S390_lowcore, 512);
	memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
	lowcore->async_stack = async_stack + ASYNC_SIZE;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;

#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE) {
		unsigned long save_area;

		save_area = get_zeroed_page(GFP_KERNEL);
		if (!save_area)
			goto out_save_area;
		lowcore->extended_save_area_addr = (u32) save_area;
	}
#endif
	lowcore_ptr[cpu] = lowcore;
	return 0;

#ifndef CONFIG_64BIT
out_save_area:
	free_page(panic_stack);
#endif
653
out:
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
	free_pages(async_stack, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	return -ENOMEM;
}

#ifdef CONFIG_HOTPLUG_CPU
static void smp_free_lowcore(int cpu)
{
	struct _lowcore *lowcore;
	int lc_order;

	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = lowcore_ptr[cpu];
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		free_page((unsigned long) lowcore->extended_save_area_addr);
#endif
	free_page(lowcore->panic_stack - PAGE_SIZE);
	free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
	free_pages((unsigned long) lowcore, lc_order);
	lowcore_ptr[cpu] = NULL;
}
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
678
/* Upping and downing of CPUs */
679
int __cpuinit __cpu_up(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
680
681
{
	struct task_struct *idle;
682
	struct _lowcore *cpu_lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
683
	struct stack_frame *sf;
684
	sigp_ccode ccode;
Linus Torvalds's avatar
Linus Torvalds committed
685

686
687
	if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
		return -EIO;
688
689
	if (smp_alloc_lowcore(cpu))
		return -ENOMEM;
Linus Torvalds's avatar
Linus Torvalds committed
690
691
692

	ccode = signal_processor_p((__u32)(unsigned long)(lowcore_ptr[cpu]),
				   cpu, sigp_set_prefix);
693
	if (ccode) {
Linus Torvalds's avatar
Linus Torvalds committed
694
695
696
697
698
699
700
		printk("sigp_set_prefix failed for cpu %d "
		       "with condition code %d\n",
		       (int) cpu, (int) ccode);
		return -EIO;
	}

	idle = current_set[cpu];
701
	cpu_lowcore = lowcore_ptr[cpu];
Linus Torvalds's avatar
Linus Torvalds committed
702
	cpu_lowcore->kernel_stack = (unsigned long)
703
		task_stack_page(idle) + THREAD_SIZE;
704
	cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
Linus Torvalds's avatar
Linus Torvalds committed
705
706
707
708
709
710
	sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
				     - sizeof(struct pt_regs)
				     - sizeof(struct stack_frame));
	memset(sf, 0, sizeof(struct stack_frame));
	sf->gprs[9] = (unsigned long) sf;
	cpu_lowcore->save_area[15] = (unsigned long) sf;
711
	__ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
712
713
714
	asm volatile(
		"	stam	0,15,0(%0)"
		: : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
Linus Torvalds's avatar
Linus Torvalds committed
715
	cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
716
717
	cpu_lowcore->current_task = (unsigned long) idle;
	cpu_lowcore->cpu_data.cpu_nr = cpu;
718
719
	cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
	cpu_lowcore->ipl_device = S390_lowcore.ipl_device;
Linus Torvalds's avatar
Linus Torvalds committed
720
	eieio();
Michael Ryan's avatar
Michael Ryan committed
721

722
	while (signal_processor(cpu, sigp_restart) == sigp_busy)
Michael Ryan's avatar
Michael Ryan committed
723
		udelay(10);
Linus Torvalds's avatar
Linus Torvalds committed
724
725
726
727
728
729

	while (!cpu_online(cpu))
		cpu_relax();
	return 0;
}

730
static int __init setup_possible_cpus(char *s)
731
{
732
	int pcpus, cpu;
733

734
735
736
	pcpus = simple_strtoul(s, NULL, 0);
	cpu_possible_map = cpumask_of_cpu(0);
	for (cpu = 1; cpu < pcpus && cpu < NR_CPUS; cpu++)
737
		cpu_set(cpu, cpu_possible_map);
738
739
740
741
	return 0;
}
early_param("possible_cpus", setup_possible_cpus);

742
743
#ifdef CONFIG_HOTPLUG_CPU

744
int __cpu_disable(void)
Linus Torvalds's avatar
Linus Torvalds committed
745
{
746
	struct ec_creg_mask_parms cr_parms;
Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
747
	int cpu = smp_processor_id();
Linus Torvalds's avatar
Linus Torvalds committed
748

Zwane Mwaikambo's avatar
Zwane Mwaikambo committed
749
	cpu_clear(cpu, cpu_online_map);
Linus Torvalds's avatar
Linus Torvalds committed
750
751

	/* Disable pfault pseudo page faults on this cpu. */
Heiko Carstens's avatar
Heiko Carstens committed
752
	pfault_fini();
Linus Torvalds's avatar
Linus Torvalds committed
753

754
755
	memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
	memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
Linus Torvalds's avatar
Linus Torvalds committed
756

757
	/* disable all external interrupts */
Linus Torvalds's avatar
Linus Torvalds committed
758
	cr_parms.orvals[0] = 0;
759
760
	cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 12 |
				1 << 11 | 1 << 10 | 1 <<  6 | 1 <<  4);
Linus Torvalds's avatar
Linus Torvalds committed
761
762
	/* disable all I/O interrupts */
	cr_parms.orvals[6] = 0;
763
764
	cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
				1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
Linus Torvalds's avatar
Linus Torvalds committed
765
766
	/* disable most machine checks */
	cr_parms.orvals[14] = 0;
767
768
	cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
				 1 << 25 | 1 << 24);
769

Linus Torvalds's avatar
Linus Torvalds committed
770
771
772
773
774
	smp_ctl_bit_callback(&cr_parms);

	return 0;
}

775
void __cpu_die(unsigned int cpu)
Linus Torvalds's avatar
Linus Torvalds committed
776
777
778
779
{
	/* Wait until target cpu is down */
	while (!smp_cpu_not_running(cpu))
		cpu_relax();
780
	smp_free_lowcore(cpu);
781
	printk(KERN_INFO "Processor %d spun down\n", cpu);
Linus Torvalds's avatar
Linus Torvalds committed
782
783
}

784
void cpu_die(void)
Linus Torvalds's avatar
Linus Torvalds committed
785
786
787
788
{
	idle_task_exit();
	signal_processor(smp_processor_id(), sigp_stop);
	BUG();
789
	for (;;);
Linus Torvalds's avatar
Linus Torvalds committed
790
791
}

792
793
#endif /* CONFIG_HOTPLUG_CPU */

Linus Torvalds's avatar
Linus Torvalds committed
794
795
void __init smp_prepare_cpus(unsigned int max_cpus)
{
796
797
798
799
800
#ifndef CONFIG_64BIT
	unsigned long save_area = 0;
#endif
	unsigned long async_stack, panic_stack;
	struct _lowcore *lowcore;
Linus Torvalds's avatar
Linus Torvalds committed
801
	unsigned int cpu;
802
	int lc_order;
803

804
805
	smp_detect_cpus();

806
807
808
	/* request the 0x1201 emergency signal external interrupt */
	if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
		panic("Couldn't request external interrupt 0x1201");
Linus Torvalds's avatar
Linus Torvalds committed
809
810
	print_cpu_info(&S390_lowcore.cpu_data);

811
812
813
814
815
	/* Reallocate current lowcore, but keep its contents. */
	lc_order = sizeof(long) == 8 ? 1 : 0;
	lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, lc_order);
	panic_stack = __get_free_page(GFP_KERNEL);
	async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
816
#ifndef CONFIG_64BIT
817
	if (MACHINE_HAS_IEEE)
818
		save_area = get_zeroed_page(GFP_KERNEL);
819
#endif
820
821
822
823
824
825
826
827
828
829
830
831
832
	local_irq_disable();
	local_mcck_disable();
	lowcore_ptr[smp_processor_id()] = lowcore;
	*lowcore = S390_lowcore;
	lowcore->panic_stack = panic_stack + PAGE_SIZE;
	lowcore->async_stack = async_stack + ASYNC_SIZE;
#ifndef CONFIG_64BIT
	if (MACHINE_HAS_IEEE)
		lowcore->extended_save_area_addr = (u32) save_area;
#endif
	set_prefix((u32)(unsigned long) lowcore);
	local_mcck_enable();
	local_irq_enable();
833
	for_each_possible_cpu(cpu)
Linus Torvalds's avatar
Linus Torvalds committed
834
835
836
837
		if (cpu != smp_processor_id())
			smp_create_idle(cpu);
}

838
void __init smp_prepare_boot_cpu(void)
Linus Torvalds's avatar
Linus Torvalds committed
839
840
841
{
	BUG_ON(smp_processor_id() != 0);

842
843
	current_thread_info()->cpu = 0;
	cpu_set(0, cpu_present_map);
Linus Torvalds's avatar
Linus Torvalds committed
844
845
846
	cpu_set(0, cpu_online_map);
	S390_lowcore.percpu_offset = __per_cpu_offset[0];
	current_set[0] = current;
847
	smp_cpu_state[0] = CPU_STATE_CONFIGURED;
848
	smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
Linus Torvalds's avatar
Linus Torvalds committed
849
850
}

851
void __init smp_cpus_done(unsigned int max_cpus)
Linus Torvalds's avatar
Linus Torvalds committed
852
853
854
855
856
857
858
859
860
861
862
{
}

/*
 * the frequency of the profiling timer can be changed
 * by writing a multiplier value into /proc/profile.
 *
 * usually you want to run this on all CPUs ;)
 */
int setup_profiling_timer(unsigned int multiplier)
{
863
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
864
865
}

866
#ifdef CONFIG_HOTPLUG_CPU
867
868
static ssize_t cpu_configure_show(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
869
870
871
872
873
874
875
876
877
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

878
879
880
static ssize_t cpu_configure_store(struct sys_device *dev,
				  struct sysdev_attribute *attr,
				  const char *buf, size_t count)
881
882
883
884
885
886
887
888
889
890
{
	int cpu = dev->id;
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;

891
	get_online_cpus();
892
	mutex_lock(&smp_cpu_state_mutex);
893
894
895
896
897
898
899
900
	rc = -EBUSY;
	if (cpu_online(cpu))
		goto out;
	rc = 0;
	switch (val) {
	case 0:
		if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
			rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
901
			if (!rc) {
902
				smp_cpu_state[cpu] = CPU_STATE_STANDBY;
903
904
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
905
906
907
908
909
		}
		break;
	case 1:
		if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
			rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
910
			if (!rc) {
911
				smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
912
913
				smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
			}
914
915
916
917
918
919
920
		}
		break;
	default:
		break;
	}
out:
	mutex_unlock(&smp_cpu_state_mutex);
921
	put_online_cpus();
922
923
924
925
926
	return rc ? rc : count;
}
static SYSDEV_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
#endif /* CONFIG_HOTPLUG_CPU */

927
928
static ssize_t cpu_polarization_show(struct sys_device *dev,
				     struct sysdev_attribute *attr, char *buf)
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
{
	int cpu = dev->id;
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	switch (smp_cpu_polarization[cpu]) {
	case POLARIZATION_HRZ:
		count = sprintf(buf, "horizontal\n");
		break;
	case POLARIZATION_VL:
		count = sprintf(buf, "vertical:low\n");
		break;
	case POLARIZATION_VM:
		count = sprintf(buf, "vertical:medium\n");
		break;
	case POLARIZATION_VH:
		count = sprintf(buf, "vertical:high\n");
		break;
	default:
		count = sprintf(buf, "unknown\n");
		break;
	}
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}
static SYSDEV_ATTR(polarization, 0444, cpu_polarization_show, NULL);

956
957
static ssize_t show_cpu_address(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
958
959
960
961
962
963
964
965
966
967
968
{
	return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
}
static SYSDEV_ATTR(address, 0444, show_cpu_address, NULL);


static struct attribute *cpu_common_attrs[] = {
#ifdef CONFIG_HOTPLUG_CPU
	&attr_configure.attr,
#endif
	&attr_address.attr,
969
	&attr_polarization.attr,
970
971
972
973
974
975
	NULL,
};

static struct attribute_group cpu_common_attr_group = {
	.attrs = cpu_common_attrs,
};
Linus Torvalds's avatar
Linus Torvalds committed
976

977
978
static ssize_t show_capability(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
979
980
981
982
983
984
985
986
987
988
989
{
	unsigned int capability;
	int rc;

	rc = get_cpu_capability(&capability);
	if (rc)
		return rc;
	return sprintf(buf, "%u\n", capability);
}
static SYSDEV_ATTR(capability, 0444, show_capability, NULL);

990
991
static ssize_t show_idle_count(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
992
993
994
995
996
997
998
999
1000
1001
1002
1003
{
	struct s390_idle_data *idle;
	unsigned long long idle_count;

	idle = &per_cpu(s390_idle, dev->id);
	spin_lock_irq(&idle->lock);
	idle_count = idle->idle_count;
	spin_unlock_irq(&idle->lock);
	return sprintf(buf, "%llu\n", idle_count);
}
static SYSDEV_ATTR(idle_count, 0444, show_idle_count, NULL);

1004
1005
static ssize_t show_idle_time(struct sys_device *dev,
				struct sysdev_attribute *attr, char *buf)
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
{
	struct s390_idle_data *idle;
	unsigned long long new_time;

	idle = &per_cpu(s390_idle, dev->id);
	spin_lock_irq(&idle->lock);
	if (idle->in_idle) {
		new_time = get_clock();
		idle->idle_time += new_time - idle->idle_enter;
		idle->idle_enter = new_time;
	}
	new_time = idle->idle_time;
	spin_unlock_irq(&idle->lock);
1019
	return sprintf(buf, "%llu\n", new_time >> 12);
1020
}
1021
static SYSDEV_ATTR(idle_time_us, 0444, show_idle_time, NULL);
1022

1023
static struct attribute *cpu_online_attrs[] = {
1024
1025
	&attr_capability.attr,
	&attr_idle_count.attr,
1026
	&attr_idle_time_us.attr,
1027
1028
1029
	NULL,
};

1030
1031
static struct attribute_group cpu_online_attr_group = {
	.attrs = cpu_online_attrs,
1032
1033
};

1034
1035
1036
1037
1038
1039
static int __cpuinit smp_cpu_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
	unsigned int cpu = (unsigned int)(long)hcpu;
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
1040
	struct s390_idle_data *idle;
1041
1042
1043

	switch (action) {
	case CPU_ONLINE:
1044
	case CPU_ONLINE_FROZEN:
1045
1046
1047
1048
1049
1050
		idle = &per_cpu(s390_idle, cpu);
		spin_lock_irq(&idle->lock);
		idle->idle_enter = 0;
		idle->idle_time = 0;
		idle->idle_count = 0;
		spin_unlock_irq(&idle->lock);
1051
		if (sysfs_create_group(&s->kobj, &cpu_online_attr_group))
1052
1053
1054
			return NOTIFY_BAD;
		break;
	case CPU_DEAD:
1055
	case CPU_DEAD_FROZEN:
1056
		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1057
1058
1059
1060
1061
1062
		break;
	}
	return NOTIFY_OK;
}

static struct notifier_block __cpuinitdata smp_cpu_nb = {
1063
	.notifier_call = smp_cpu_notify,
1064
1065
};

1066
static int __devinit smp_add_present_cpu(int cpu)
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
{
	struct cpu *c = &per_cpu(cpu_devices, cpu);
	struct sys_device *s = &c->sysdev;
	int rc;

	c->hotpluggable = 1;
	rc = register_cpu(c, cpu);
	if (rc)
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
	if (rc)
		goto out_cpu;
	if (!cpu_online(cpu))
		goto out;
	rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
	if (!rc)
		return 0;
	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
out_cpu:
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu(c);
#endif
out:
	return rc;
}

#ifdef CONFIG_HOTPLUG_CPU
1094

1095
int __ref smp_rescan_cpus(void)
1096
1097
1098
1099
1100
{
	cpumask_t newcpus;
	int cpu;
	int rc;

1101
	get_online_cpus();
1102
	mutex_lock(&smp_cpu_state_mutex);
1103
	newcpus = cpu_present_map;
1104
	rc = __smp_rescan_cpus();
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
	if (rc)
		goto out;
	cpus_andnot(newcpus, cpu_present_map, newcpus);
	for_each_cpu_mask(cpu, newcpus) {
		rc = smp_add_present_cpu(cpu);
		if (rc)
			cpu_clear(cpu, cpu_present_map);
	}
	rc = 0;
out:
	mutex_unlock(&smp_cpu_state_mutex);
1116
	put_online_cpus();
1117
1118
	if (!cpus_empty(newcpus))
		topology_schedule_update();
1119
1120
1121
	return rc;
}

1122
1123
1124
static ssize_t __ref rescan_store(struct sys_device *dev,
				  struct sysdev_attribute *attr,
				  const char *buf,
1125
1126
1127
1128
1129
				  size_t count)
{
	int rc;

	rc = smp_rescan_cpus();
1130
1131
1132
1133
1134
	return rc ? rc : count;
}
static SYSDEV_ATTR(rescan, 0200, NULL, rescan_store);
#endif /* CONFIG_HOTPLUG_CPU */

1135
1136
1137
static ssize_t dispatching_show(struct sys_device *dev,
				struct sysdev_attribute *attr,
				char *buf)
1138
1139
1140
1141
1142
1143
1144
1145
1146
{
	ssize_t count;

	mutex_lock(&smp_cpu_state_mutex);
	count = sprintf(buf, "%d\n", cpu_management);
	mutex_unlock(&smp_cpu_state_mutex);
	return count;
}

1147
1148
1149
static ssize_t dispatching_store(struct sys_device *dev,
				 struct sysdev_attribute *attr,
				 const char *buf, size_t count)
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
{
	int val, rc;
	char delim;

	if (sscanf(buf, "%d %c", &val, &delim) != 1)
		return -EINVAL;
	if (val != 0 && val != 1)
		return -EINVAL;
	rc = 0;
	get_online_cpus();
1160
	mutex_lock(&smp_cpu_state_mutex);
1161
1162
1163
1164
1165
1166
1167
	if (cpu_management == val)
		goto out;
	rc = topology_set_cpu_management(val);
	if (!rc)
		cpu_management = val;
out:
	mutex_unlock(&smp_cpu_state_mutex);
1168
	put_online_cpus();
1169
1170
1171
1172
	return rc ? rc : count;
}
static SYSDEV_ATTR(dispatching, 0644, dispatching_show, dispatching_store);

Linus Torvalds's avatar
Linus Torvalds committed
1173
1174
1175
static int __init topology_init(void)
{
	int cpu;
1176
	int rc;
1177
1178

	register_cpu_notifier(&smp_cpu_nb);
Linus Torvalds's avatar
Linus Torvalds committed
1179

1180
1181
1182
1183
1184
1185
#ifdef CONFIG_HOTPLUG_CPU
	rc = sysfs_create_file(&cpu_sysdev_class.kset.kobj,
			       &attr_rescan.attr);
	if (rc)
		return rc;
#endif
1186
1187
1188
1189
	rc = sysfs_create_file(&cpu_sysdev_class.kset.kobj,
			       &attr_dispatching.attr);
	if (rc)
		return rc;
1190
1191
	for_each_present_cpu(cpu) {
		rc = smp_add_present_cpu(cpu);
1192
1193
		if (rc)
			return rc;
Linus Torvalds's avatar
Linus Torvalds committed
1194
1195
1196
1197
	}
	return 0;
}
subsys_initcall(topology_init);