stop_machine.c 17.8 KB
Newer Older
Tejun Heo's avatar
Tejun Heo committed
1
2
3
4
5
6
7
8
9
/*
 * kernel/stop_machine.c
 *
 * Copyright (C) 2008, 2005	IBM Corporation.
 * Copyright (C) 2008, 2005	Rusty Russell rusty@rustcorp.com.au
 * Copyright (C) 2010		SUSE Linux Products GmbH
 * Copyright (C) 2010		Tejun Heo <tj@kernel.org>
 *
 * This file is released under the GPLv2 and any later version.
10
 */
Tejun Heo's avatar
Tejun Heo committed
11
#include <linux/completion.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <linux/cpu.h>
Tejun Heo's avatar
Tejun Heo committed
13
#include <linux/init.h>
14
#include <linux/kthread.h>
15
#include <linux/export.h>
Tejun Heo's avatar
Tejun Heo committed
16
#include <linux/percpu.h>
17
18
#include <linux/sched.h>
#include <linux/stop_machine.h>
19
#include <linux/interrupt.h>
Tejun Heo's avatar
Tejun Heo committed
20
#include <linux/kallsyms.h>
21
#include <linux/smpboot.h>
Arun Sharma's avatar
Arun Sharma committed
22
#include <linux/atomic.h>
23
#include <linux/nmi.h>
24
#include <linux/sched/wake_q.h>
Tejun Heo's avatar
Tejun Heo committed
25
26
27
28
29
30
31
32
33
34
35
36
37

/*
 * Structure to determine completion condition and record errors.  May
 * be shared by works on different cpus.
 */
struct cpu_stop_done {
	atomic_t		nr_todo;	/* nr left to execute */
	int			ret;		/* collected return value */
	struct completion	completion;	/* fired if nr_todo reaches 0 */
};

/* the actual stopper, one per every possible cpu, enabled on online cpus */
struct cpu_stopper {
38
39
	struct task_struct	*thread;

40
	raw_spinlock_t		lock;
41
	bool			enabled;	/* is this stopper enabled? */
Tejun Heo's avatar
Tejun Heo committed
42
	struct list_head	works;		/* list of pending works */
43
44

	struct cpu_stop_work	stop_work;	/* for stop_cpus */
Tejun Heo's avatar
Tejun Heo committed
45
46
47
};

static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
48
static bool stop_machine_initialized = false;
Tejun Heo's avatar
Tejun Heo committed
49

50
51
52
/* static data for stop_cpus */
static DEFINE_MUTEX(stop_cpus_mutex);
static bool stop_cpus_in_progress;
53

Tejun Heo's avatar
Tejun Heo committed
54
55
56
57
58
59
60
61
static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
{
	memset(done, 0, sizeof(*done));
	atomic_set(&done->nr_todo, nr_todo);
	init_completion(&done->completion);
}

/* signal completion unless @done is NULL */
62
static void cpu_stop_signal_done(struct cpu_stop_done *done)
Tejun Heo's avatar
Tejun Heo committed
63
{
64
65
	if (atomic_dec_and_test(&done->nr_todo))
		complete(&done->completion);
Tejun Heo's avatar
Tejun Heo committed
66
67
}

68
static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
69
70
					struct cpu_stop_work *work,
					struct wake_q_head *wakeq)
71
72
{
	list_add_tail(&work->list, &stopper->works);
73
	wake_q_add(wakeq, stopper->thread);
74
75
}

Tejun Heo's avatar
Tejun Heo committed
76
/* queue @work to @stopper.  if offline, @work is completed immediately */
77
static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
Tejun Heo's avatar
Tejun Heo committed
78
{
79
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
80
	DEFINE_WAKE_Q(wakeq);
Tejun Heo's avatar
Tejun Heo committed
81
	unsigned long flags;
82
	bool enabled;
Tejun Heo's avatar
Tejun Heo committed
83

84
	preempt_disable();
85
	raw_spin_lock_irqsave(&stopper->lock, flags);
86
87
	enabled = stopper->enabled;
	if (enabled)
88
		__cpu_stop_queue_work(stopper, work, &wakeq);
89
	else if (work->done)
90
		cpu_stop_signal_done(work->done);
91
	raw_spin_unlock_irqrestore(&stopper->lock, flags);
92

93
	wake_up_q(&wakeq);
94
	preempt_enable();
95

96
	return enabled;
Tejun Heo's avatar
Tejun Heo committed
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
}

/**
 * stop_one_cpu - stop a cpu
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on @cpu.  @fn is run in a process context with
 * the highest priority preempting any task on the cpu and
 * monopolizing it.  This function returns after the execution is
 * complete.
 *
 * This function doesn't guarantee @cpu stays online till @fn
 * completes.  If @cpu goes down in the middle, execution may happen
 * partially or fully on different cpus.  @fn should either be ready
 * for that or the caller should ensure that @cpu stays online until
 * this function completes.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
 * otherwise, the return value of @fn.
 */
int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
{
	struct cpu_stop_done done;
	struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };

	cpu_stop_init_done(&done, 1);
129
130
	if (!cpu_stop_queue_work(cpu, &work))
		return -ENOENT;
131
132
133
134
135
	/*
	 * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
	 * cycle by doing a preemption:
	 */
	cond_resched();
Tejun Heo's avatar
Tejun Heo committed
136
	wait_for_completion(&done.completion);
137
	return done.ret;
Tejun Heo's avatar
Tejun Heo committed
138
139
}

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
/* This controls the threads on each CPU. */
enum multi_stop_state {
	/* Dummy starting state for thread. */
	MULTI_STOP_NONE,
	/* Awaiting everyone to be scheduled. */
	MULTI_STOP_PREPARE,
	/* Disable interrupts. */
	MULTI_STOP_DISABLE_IRQ,
	/* Run the function */
	MULTI_STOP_RUN,
	/* Exit */
	MULTI_STOP_EXIT,
};

struct multi_stop_data {
155
	cpu_stop_fn_t		fn;
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
	void			*data;
	/* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
	unsigned int		num_threads;
	const struct cpumask	*active_cpus;

	enum multi_stop_state	state;
	atomic_t		thread_ack;
};

static void set_state(struct multi_stop_data *msdata,
		      enum multi_stop_state newstate)
{
	/* Reset ack counter. */
	atomic_set(&msdata->thread_ack, msdata->num_threads);
	smp_wmb();
	msdata->state = newstate;
}

/* Last one to ack a state moves to the next state. */
static void ack_state(struct multi_stop_data *msdata)
{
	if (atomic_dec_and_test(&msdata->thread_ack))
		set_state(msdata, msdata->state + 1);
}

/* This is the cpu_stop function which stops the CPU. */
static int multi_cpu_stop(void *data)
{
	struct multi_stop_data *msdata = data;
	enum multi_stop_state curstate = MULTI_STOP_NONE;
	int cpu = smp_processor_id(), err = 0;
	unsigned long flags;
	bool is_active;

	/*
	 * When called from stop_machine_from_inactive_cpu(), irq might
	 * already be disabled.  Save the state and restore it on exit.
	 */
	local_save_flags(flags);

	if (!msdata->active_cpus)
		is_active = cpu == cpumask_first(cpu_online_mask);
	else
		is_active = cpumask_test_cpu(cpu, msdata->active_cpus);

	/* Simple state machine */
	do {
		/* Chill out and ensure we re-read multi_stop_state. */
204
		cpu_relax_yield();
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
		if (msdata->state != curstate) {
			curstate = msdata->state;
			switch (curstate) {
			case MULTI_STOP_DISABLE_IRQ:
				local_irq_disable();
				hard_irq_disable();
				break;
			case MULTI_STOP_RUN:
				if (is_active)
					err = msdata->fn(msdata->data);
				break;
			default:
				break;
			}
			ack_state(msdata);
220
221
222
223
224
225
226
		} else if (curstate > MULTI_STOP_PREPARE) {
			/*
			 * At this stage all other CPUs we depend on must spin
			 * in the same loop. Any reason for hard-lockup should
			 * be detected and reported on their side.
			 */
			touch_nmi_watchdog();
227
228
229
		}
	} while (curstate != MULTI_STOP_EXIT);

230
	hard_irq_enable();
231
232
233
234
	local_irq_restore(flags);
	return err;
}

235
236
237
static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
				    int cpu2, struct cpu_stop_work *work2)
{
238
239
	struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
	struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
240
	DEFINE_WAKE_Q(wakeq);
241
	int err;
242

243
retry:
244
245
246
247
248
249
250
251
	/*
	 * The waking up of stopper threads has to happen in the same
	 * scheduling context as the queueing.  Otherwise, there is a
	 * possibility of one of the above stoppers being woken up by another
	 * CPU, and preempting us. This will cause us to not wake up the other
	 * stopper forever.
	 */
	preempt_disable();
252
253
	raw_spin_lock_irq(&stopper1->lock);
	raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
254

255
256
	if (!stopper1->enabled || !stopper2->enabled) {
		err = -ENOENT;
257
		goto unlock;
258
259
	}

260
261
262
263
264
265
266
267
268
269
	/*
	 * Ensure that if we race with __stop_cpus() the stoppers won't get
	 * queued up in reverse order leading to system deadlock.
	 *
	 * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has
	 * queued a work on cpu1 but not on cpu2, we hold both locks.
	 *
	 * It can be falsely true but it is safe to spin until it is cleared,
	 * queue_stop_cpus_work() does everything under preempt_disable().
	 */
270
271
272
273
	if (unlikely(stop_cpus_in_progress)) {
		err = -EDEADLK;
		goto unlock;
	}
274
275

	err = 0;
276
277
	__cpu_stop_queue_work(stopper1, work1, &wakeq);
	__cpu_stop_queue_work(stopper2, work2, &wakeq);
278

279
unlock:
280
281
	raw_spin_unlock(&stopper2->lock);
	raw_spin_unlock_irq(&stopper1->lock);
282

283
	if (unlikely(err == -EDEADLK)) {
284
285
		preempt_enable();

286
287
		while (stop_cpus_in_progress)
			cpu_relax();
288

289
290
		goto retry;
	}
291

292
293
	wake_up_q(&wakeq);
	preempt_enable();
294

295
	return err;
296
}
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
/**
 * stop_two_cpus - stops two cpus
 * @cpu1: the cpu to stop
 * @cpu2: the other cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Stops both the current and specified CPU and runs @fn on one of them.
 *
 * returns when both are completed.
 */
int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
{
	struct cpu_stop_done done;
	struct cpu_stop_work work1, work2;
312
313
314
	struct multi_stop_data msdata;

	msdata = (struct multi_stop_data){
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
		.fn = fn,
		.data = arg,
		.num_threads = 2,
		.active_cpus = cpumask_of(cpu1),
	};

	work1 = work2 = (struct cpu_stop_work){
		.fn = multi_cpu_stop,
		.arg = &msdata,
		.done = &done
	};

	cpu_stop_init_done(&done, 2);
	set_state(&msdata, MULTI_STOP_PREPARE);

330
331
	if (cpu1 > cpu2)
		swap(cpu1, cpu2);
332
	if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
333
		return -ENOENT;
334
335

	wait_for_completion(&done.completion);
336
	return done.ret;
337
338
}

Tejun Heo's avatar
Tejun Heo committed
339
340
341
342
343
/**
 * stop_one_cpu_nowait - stop a cpu but don't wait for completion
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
344
 * @work_buf: pointer to cpu_stop_work structure
Tejun Heo's avatar
Tejun Heo committed
345
346
347
348
349
350
351
 *
 * Similar to stop_one_cpu() but doesn't wait for completion.  The
 * caller is responsible for ensuring @work_buf is currently unused
 * and will remain untouched until stopper starts executing @fn.
 *
 * CONTEXT:
 * Don't care.
352
353
354
355
 *
 * RETURNS:
 * true if cpu_stop_work was queued successfully and @fn will be called,
 * false otherwise.
Tejun Heo's avatar
Tejun Heo committed
356
 */
357
bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
Tejun Heo's avatar
Tejun Heo committed
358
359
360
			struct cpu_stop_work *work_buf)
{
	*work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
361
	return cpu_stop_queue_work(cpu, work_buf);
Tejun Heo's avatar
Tejun Heo committed
362
363
}

364
static bool queue_stop_cpus_work(const struct cpumask *cpumask,
365
366
				 cpu_stop_fn_t fn, void *arg,
				 struct cpu_stop_done *done)
Tejun Heo's avatar
Tejun Heo committed
367
368
369
{
	struct cpu_stop_work *work;
	unsigned int cpu;
370
	bool queued = false;
Tejun Heo's avatar
Tejun Heo committed
371
372
373
374
375
376

	/*
	 * Disable preemption while queueing to avoid getting
	 * preempted by a stopper which might wait for other stoppers
	 * to enter @fn which can lead to deadlock.
	 */
377
378
	preempt_disable();
	stop_cpus_in_progress = true;
379
380
381
382
383
	for_each_cpu(cpu, cpumask) {
		work = &per_cpu(cpu_stopper.stop_work, cpu);
		work->fn = fn;
		work->arg = arg;
		work->done = done;
384
385
		if (cpu_stop_queue_work(cpu, work))
			queued = true;
386
	}
387
388
	stop_cpus_in_progress = false;
	preempt_enable();
389
390

	return queued;
391
}
Tejun Heo's avatar
Tejun Heo committed
392

393
394
395
396
397
398
static int __stop_cpus(const struct cpumask *cpumask,
		       cpu_stop_fn_t fn, void *arg)
{
	struct cpu_stop_done done;

	cpu_stop_init_done(&done, cpumask_weight(cpumask));
399
400
	if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
		return -ENOENT;
Tejun Heo's avatar
Tejun Heo committed
401
	wait_for_completion(&done.completion);
402
	return done.ret;
Tejun Heo's avatar
Tejun Heo committed
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
}

/**
 * stop_cpus - stop multiple cpus
 * @cpumask: cpus to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on online cpus in @cpumask.  On each target cpu,
 * @fn is run in a process context with the highest priority
 * preempting any task on the cpu and monopolizing it.  This function
 * returns after all executions are complete.
 *
 * This function doesn't guarantee the cpus in @cpumask stay online
 * till @fn completes.  If some cpus go down in the middle, execution
 * on the cpu may happen partially or fully on different cpus.  @fn
 * should either be ready for that or the caller should ensure that
 * the cpus stay online until this function completes.
 *
 * All stop_cpus() calls are serialized making it safe for @fn to wait
 * for all cpus to start executing it.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed at all because all cpus in
 * @cpumask were offline; otherwise, 0 if all executions of @fn
 * returned 0, any non zero return value if any returned non zero.
 */
int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
	int ret;

	/* static works are used, process one request at a time */
	mutex_lock(&stop_cpus_mutex);
	ret = __stop_cpus(cpumask, fn, arg);
	mutex_unlock(&stop_cpus_mutex);
	return ret;
}

/**
 * try_stop_cpus - try to stop multiple cpus
 * @cpumask: cpus to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Identical to stop_cpus() except that it fails with -EAGAIN if
 * someone else is already using the facility.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -EAGAIN if someone else is already stopping cpus, -ENOENT if
 * @fn(@arg) was not executed at all because all cpus in @cpumask were
 * offline; otherwise, 0 if all executions of @fn returned 0, any non
 * zero return value if any returned non zero.
 */
int try_stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
	int ret;

	/* static works are used, process one request at a time */
	if (!mutex_trylock(&stop_cpus_mutex))
		return -EAGAIN;
	ret = __stop_cpus(cpumask, fn, arg);
	mutex_unlock(&stop_cpus_mutex);
	return ret;
}

474
475
476
477
478
479
static int cpu_stop_should_run(unsigned int cpu)
{
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
	unsigned long flags;
	int run;

480
	raw_spin_lock_irqsave(&stopper->lock, flags);
481
	run = !list_empty(&stopper->works);
482
	raw_spin_unlock_irqrestore(&stopper->lock, flags);
483
484
485
486
	return run;
}

static void cpu_stopper_thread(unsigned int cpu)
Tejun Heo's avatar
Tejun Heo committed
487
{
488
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
Tejun Heo's avatar
Tejun Heo committed
489
490
491
492
	struct cpu_stop_work *work;

repeat:
	work = NULL;
493
	raw_spin_lock_irq(&stopper->lock);
Tejun Heo's avatar
Tejun Heo committed
494
495
496
497
498
	if (!list_empty(&stopper->works)) {
		work = list_first_entry(&stopper->works,
					struct cpu_stop_work, list);
		list_del_init(&work->list);
	}
499
	raw_spin_unlock_irq(&stopper->lock);
Tejun Heo's avatar
Tejun Heo committed
500
501
502
503
504

	if (work) {
		cpu_stop_fn_t fn = work->fn;
		void *arg = work->arg;
		struct cpu_stop_done *done = work->done;
505
		int ret;
Tejun Heo's avatar
Tejun Heo committed
506

507
508
		/* cpu stop callbacks must not sleep, make in_atomic() == T */
		preempt_count_inc();
Tejun Heo's avatar
Tejun Heo committed
509
		ret = fn(arg);
510
511
512
513
514
		if (done) {
			if (ret)
				done->ret = ret;
			cpu_stop_signal_done(done);
		}
515
		preempt_count_dec();
Tejun Heo's avatar
Tejun Heo committed
516
		WARN_ONCE(preempt_count(),
517
			  "cpu_stop: %pf(%p) leaked preempt count\n", fn, arg);
518
519
		goto repeat;
	}
Tejun Heo's avatar
Tejun Heo committed
520
521
}

522
523
524
525
526
527
528
529
530
531
532
533
void stop_machine_park(int cpu)
{
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
	/*
	 * Lockless. cpu_stopper_thread() will take stopper->lock and flush
	 * the pending works before it parks, until then it is fine to queue
	 * the new works.
	 */
	stopper->enabled = false;
	kthread_park(stopper->thread);
}

534
535
extern void sched_set_stop_task(int cpu, struct task_struct *stop);

536
537
static void cpu_stop_create(unsigned int cpu)
{
538
	sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
539
540
541
}

static void cpu_stop_park(unsigned int cpu)
Tejun Heo's avatar
Tejun Heo committed
542
543
544
{
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

545
	WARN_ON(!list_empty(&stopper->works));
546
}
Tejun Heo's avatar
Tejun Heo committed
547

548
549
550
551
void stop_machine_unpark(int cpu)
{
	struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

552
	stopper->enabled = true;
553
554
555
	kthread_unpark(stopper->thread);
}

556
static struct smp_hotplug_thread cpu_stop_threads = {
557
	.store			= &cpu_stopper.thread,
558
559
560
561
562
563
	.thread_should_run	= cpu_stop_should_run,
	.thread_fn		= cpu_stopper_thread,
	.thread_comm		= "migration/%u",
	.create			= cpu_stop_create,
	.park			= cpu_stop_park,
	.selfparking		= true,
Tejun Heo's avatar
Tejun Heo committed
564
565
566
567
568
569
570
571
572
};

static int __init cpu_stop_init(void)
{
	unsigned int cpu;

	for_each_possible_cpu(cpu) {
		struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

573
		raw_spin_lock_init(&stopper->lock);
Tejun Heo's avatar
Tejun Heo committed
574
575
576
		INIT_LIST_HEAD(&stopper->works);
	}

577
	BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
578
	stop_machine_unpark(raw_smp_processor_id());
579
	stop_machine_initialized = true;
Tejun Heo's avatar
Tejun Heo committed
580
581
582
	return 0;
}
early_initcall(cpu_stop_init);
Linus Torvalds's avatar
Linus Torvalds committed
583

584
585
int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
			    const struct cpumask *cpus)
Linus Torvalds's avatar
Linus Torvalds committed
586
{
587
588
589
590
591
592
	struct multi_stop_data msdata = {
		.fn = fn,
		.data = data,
		.num_threads = num_online_cpus(),
		.active_cpus = cpus,
	};
593

594
595
	lockdep_assert_cpus_held();

596
597
598
599
600
601
602
603
604
	if (!stop_machine_initialized) {
		/*
		 * Handle the case where stop_machine() is called
		 * early in boot before stop_machine() has been
		 * initialized.
		 */
		unsigned long flags;
		int ret;

605
		WARN_ON_ONCE(msdata.num_threads != 1);
606
607
608
609

		local_irq_save(flags);
		hard_irq_disable();
		ret = (*fn)(data);
610
		hard_irq_enable();
611
612
613
614
615
		local_irq_restore(flags);

		return ret;
	}

616
	/* Set the initial state and stop all online cpus. */
617
618
	set_state(&msdata, MULTI_STOP_PREPARE);
	return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
Linus Torvalds's avatar
Linus Torvalds committed
619
620
}

621
int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
Linus Torvalds's avatar
Linus Torvalds committed
622
623
624
625
{
	int ret;

	/* No CPUs can come up or down during this. */
626
627
628
	cpus_read_lock();
	ret = stop_machine_cpuslocked(fn, data, cpus);
	cpus_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
629
630
	return ret;
}
631
EXPORT_SYMBOL_GPL(stop_machine);
632

633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
/**
 * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
 * @fn: the function to run
 * @data: the data ptr for the @fn()
 * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
 *
 * This is identical to stop_machine() but can be called from a CPU which
 * is not active.  The local CPU is in the process of hotplug (so no other
 * CPU hotplug can start) and not marked active and doesn't have enough
 * context to sleep.
 *
 * This function provides stop_machine() functionality for such state by
 * using busy-wait for synchronization and executing @fn directly for local
 * CPU.
 *
 * CONTEXT:
 * Local CPU is inactive.  Temporarily stops all active CPUs.
 *
 * RETURNS:
 * 0 if all executions of @fn returned 0, any non zero return value if any
 * returned non zero.
 */
655
int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
656
657
				  const struct cpumask *cpus)
{
658
	struct multi_stop_data msdata = { .fn = fn, .data = data,
659
660
661
662
663
664
					    .active_cpus = cpus };
	struct cpu_stop_done done;
	int ret;

	/* Local CPU must be inactive and CPU hotplug in progress. */
	BUG_ON(cpu_active(raw_smp_processor_id()));
665
	msdata.num_threads = num_active_cpus() + 1;	/* +1 for local */
666
667
668
669
670
671

	/* No proper task established and can't sleep - busy wait for lock. */
	while (!mutex_trylock(&stop_cpus_mutex))
		cpu_relax();

	/* Schedule work on other CPUs and execute directly for local CPU */
672
	set_state(&msdata, MULTI_STOP_PREPARE);
673
	cpu_stop_init_done(&done, num_active_cpus());
674
	queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
675
			     &done);
676
	ret = multi_cpu_stop(&msdata);
677
678
679
680
681
682
683
684

	/* Busy wait for completion. */
	while (!completion_done(&done.completion))
		cpu_relax();

	mutex_unlock(&stop_cpus_mutex);
	return ret ?: done.ret;
}