vmscan.c 120 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
5
6
7
8
9
10
11
12
13
14
/*
 *  linux/mm/vmscan.c
 *
 *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
 *
 *  Swap reorganised 29.12.95, Stephen Tweedie.
 *  kswapd added: 7.1.96  sct
 *  Removed kswapd_ctl limits, and swap out as many pages as needed
 *  to bring the system back to freepages.high: 2.4.97, Rik van Riel.
 *  Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
 *  Multiqueue VM started 5.8.00, Rik van Riel.
 */

15
16
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

Linus Torvalds's avatar
Linus Torvalds committed
17
#include <linux/mm.h>
18
#include <linux/sched/mm.h>
Linus Torvalds's avatar
Linus Torvalds committed
19
#include <linux/module.h>
20
#include <linux/gfp.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
23
24
25
#include <linux/kernel_stat.h>
#include <linux/swap.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/highmem.h>
26
#include <linux/vmpressure.h>
27
#include <linux/vmstat.h>
Linus Torvalds's avatar
Linus Torvalds committed
28
29
30
31
32
33
34
35
36
37
38
#include <linux/file.h>
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/buffer_head.h>	/* for try_to_release_page(),
					buffer_heads_over_limit */
#include <linux/mm_inline.h>
#include <linux/backing-dev.h>
#include <linux/rmap.h>
#include <linux/topology.h>
#include <linux/cpu.h>
#include <linux/cpuset.h>
39
#include <linux/compaction.h>
Linus Torvalds's avatar
Linus Torvalds committed
40
41
#include <linux/notifier.h>
#include <linux/rwsem.h>
42
#include <linux/delay.h>
43
#include <linux/kthread.h>
44
#include <linux/freezer.h>
45
#include <linux/memcontrol.h>
46
#include <linux/delayacct.h>
47
#include <linux/sysctl.h>
48
#include <linux/oom.h>
49
#include <linux/prefetch.h>
50
#include <linux/printk.h>
51
#include <linux/dax.h>
Linus Torvalds's avatar
Linus Torvalds committed
52
53
54
55
56

#include <asm/tlbflush.h>
#include <asm/div64.h>

#include <linux/swapops.h>
57
#include <linux/balloon_compaction.h>
Linus Torvalds's avatar
Linus Torvalds committed
58

59
60
#include "internal.h"

61
62
63
#define CREATE_TRACE_POINTS
#include <trace/events/vmscan.h>

Linus Torvalds's avatar
Linus Torvalds committed
64
struct scan_control {
65
66
67
	/* How many pages shrink_list() should reclaim */
	unsigned long nr_to_reclaim;

68
69
70
71
72
	/*
	 * Nodemask of nodes allowed by the caller. If NULL, all nodes
	 * are scanned.
	 */
	nodemask_t	*nodemask;
73

74
75
76
77
78
	/*
	 * The memory cgroup that hit its limit and as a result is the
	 * primary target of this reclaim invocation.
	 */
	struct mem_cgroup *target_mem_cgroup;
79

80
	/* Writepage batching in laptop mode; RECLAIM_WRITE */
81
82
83
84
85
86
87
88
	unsigned int may_writepage:1;

	/* Can mapped pages be reclaimed? */
	unsigned int may_unmap:1;

	/* Can pages be swapped as part of reclaim? */
	unsigned int may_swap:1;

89
90
91
92
93
94
95
	/*
	 * Cgroups are not reclaimed below their configured memory.low,
	 * unless we threaten to OOM. If any cgroups are skipped due to
	 * memory.low and nothing was reclaimed, go back for memory.low.
	 */
	unsigned int memcg_low_reclaim:1;
	unsigned int memcg_low_skipped:1;
96

97
98
99
100
101
	unsigned int hibernation_mode:1;

	/* One of the zones is ready for compaction */
	unsigned int compaction_ready:1;

102
103
104
105
106
107
108
109
110
111
112
113
	/* Allocation order */
	s8 order;

	/* Scan (total_size >> priority) pages at once */
	s8 priority;

	/* The highest zone to isolate pages for reclaim from */
	s8 reclaim_idx;

	/* This context's GFP mask */
	gfp_t gfp_mask;

114
115
116
117
118
	/* Incremented by the number of inactive pages that were scanned */
	unsigned long nr_scanned;

	/* Number of pages freed so far during a call to shrink_zones() */
	unsigned long nr_reclaimed;
119
120
121
122
123
124
125
126
127
128

	struct {
		unsigned int dirty;
		unsigned int unqueued_dirty;
		unsigned int congested;
		unsigned int writeback;
		unsigned int immediate;
		unsigned int file_taken;
		unsigned int taken;
	} nr;
Linus Torvalds's avatar
Linus Torvalds committed
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
};

#ifdef ARCH_HAS_PREFETCH
#define prefetch_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetch(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

#ifdef ARCH_HAS_PREFETCHW
#define prefetchw_prev_lru_page(_page, _base, _field)			\
	do {								\
		if ((_page)->lru.prev != _base) {			\
			struct page *prev;				\
									\
			prev = lru_to_page(&(_page->lru));		\
			prefetchw(&prev->_field);			\
		}							\
	} while (0)
#else
#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
#endif

/*
 * From 0 .. 100.  Higher means more swappy.
 */
int vm_swappiness = 60;
163
164
165
166
167
/*
 * The total number of pages which are beyond the high watermark within all
 * zones.
 */
unsigned long vm_total_pages;
Linus Torvalds's avatar
Linus Torvalds committed
168
169
170
171

static LIST_HEAD(shrinker_list);
static DECLARE_RWSEM(shrinker_rwsem);

172
#ifdef CONFIG_MEMCG_KMEM
173
174
175
176
177
178
179
180
181
182
183
184
185
186

/*
 * We allow subsystems to populate their shrinker-related
 * LRU lists before register_shrinker_prepared() is called
 * for the shrinker, since we don't want to impose
 * restrictions on their internal registration order.
 * In this case shrink_slab_memcg() may find corresponding
 * bit is set in the shrinkers map.
 *
 * This value is used by the function to detect registering
 * shrinkers and to skip do_shrink_slab() calls for them.
 */
#define SHRINKER_REGISTERING ((struct shrinker *)~0UL)

187
188
189
190
191
192
193
194
195
static DEFINE_IDR(shrinker_idr);
static int shrinker_nr_max;

static int prealloc_memcg_shrinker(struct shrinker *shrinker)
{
	int id, ret = -ENOMEM;

	down_write(&shrinker_rwsem);
	/* This may call shrinker, so it must use down_read_trylock() */
196
	id = idr_alloc(&shrinker_idr, SHRINKER_REGISTERING, 0, 0, GFP_KERNEL);
197
198
199
	if (id < 0)
		goto unlock;

200
201
202
203
204
205
	if (id >= shrinker_nr_max) {
		if (memcg_expand_shrinker_maps(id)) {
			idr_remove(&shrinker_idr, id);
			goto unlock;
		}

206
		shrinker_nr_max = id + 1;
207
	}
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
	shrinker->id = id;
	ret = 0;
unlock:
	up_write(&shrinker_rwsem);
	return ret;
}

static void unregister_memcg_shrinker(struct shrinker *shrinker)
{
	int id = shrinker->id;

	BUG_ON(id < 0);

	down_write(&shrinker_rwsem);
	idr_remove(&shrinker_idr, id);
	up_write(&shrinker_rwsem);
}
#else /* CONFIG_MEMCG_KMEM */
static int prealloc_memcg_shrinker(struct shrinker *shrinker)
{
	return 0;
}

static void unregister_memcg_shrinker(struct shrinker *shrinker)
{
}
#endif /* CONFIG_MEMCG_KMEM */

Andrew Morton's avatar
Andrew Morton committed
236
#ifdef CONFIG_MEMCG
237
238
static bool global_reclaim(struct scan_control *sc)
{
239
	return !sc->target_mem_cgroup;
240
}
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261

/**
 * sane_reclaim - is the usual dirty throttling mechanism operational?
 * @sc: scan_control in question
 *
 * The normal page dirty throttling mechanism in balance_dirty_pages() is
 * completely broken with the legacy memcg and direct stalling in
 * shrink_page_list() is used for throttling instead, which lacks all the
 * niceties such as fairness, adaptive pausing, bandwidth proportional
 * allocation and configurability.
 *
 * This function tests whether the vmscan currently in progress can assume
 * that the normal dirty throttling mechanism is operational.
 */
static bool sane_reclaim(struct scan_control *sc)
{
	struct mem_cgroup *memcg = sc->target_mem_cgroup;

	if (!memcg)
		return true;
#ifdef CONFIG_CGROUP_WRITEBACK
262
	if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
263
264
265
266
		return true;
#endif
	return false;
}
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289

static void set_memcg_congestion(pg_data_t *pgdat,
				struct mem_cgroup *memcg,
				bool congested)
{
	struct mem_cgroup_per_node *mn;

	if (!memcg)
		return;

	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
	WRITE_ONCE(mn->congested, congested);
}

static bool memcg_congested(pg_data_t *pgdat,
			struct mem_cgroup *memcg)
{
	struct mem_cgroup_per_node *mn;

	mn = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
	return READ_ONCE(mn->congested);

}
290
#else
291
292
293
294
static bool global_reclaim(struct scan_control *sc)
{
	return true;
}
295
296
297
298
299

static bool sane_reclaim(struct scan_control *sc)
{
	return true;
}
300
301
302
303
304
305
306
307
308
309
310
311

static inline void set_memcg_congestion(struct pglist_data *pgdat,
				struct mem_cgroup *memcg, bool congested)
{
}

static inline bool memcg_congested(struct pglist_data *pgdat,
			struct mem_cgroup *memcg)
{
	return false;

}
312
313
#endif

314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
/*
 * This misses isolated pages which are not accounted for to save counters.
 * As the data only determines if reclaim or compaction continues, it is
 * not expected that isolated pages will be a dominating factor.
 */
unsigned long zone_reclaimable_pages(struct zone *zone)
{
	unsigned long nr;

	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
	if (get_nr_swap_pages() > 0)
		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);

	return nr;
}

332
333
334
335
336
337
338
/**
 * lruvec_lru_size -  Returns the number of pages on the given LRU list.
 * @lruvec: lru vector
 * @lru: lru to use
 * @zone_idx: zones to consider (use MAX_NR_ZONES for the whole LRU list)
 */
unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
339
{
340
341
342
	unsigned long lru_size;
	int zid;

343
	if (!mem_cgroup_disabled())
344
345
346
		lru_size = mem_cgroup_get_lru_size(lruvec, lru);
	else
		lru_size = node_page_state(lruvec_pgdat(lruvec), NR_LRU_BASE + lru);
347

348
349
350
	for (zid = zone_idx + 1; zid < MAX_NR_ZONES; zid++) {
		struct zone *zone = &lruvec_pgdat(lruvec)->node_zones[zid];
		unsigned long size;
351

352
353
354
355
356
357
358
359
360
361
362
363
		if (!managed_zone(zone))
			continue;

		if (!mem_cgroup_disabled())
			size = mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
		else
			size = zone_page_state(&lruvec_pgdat(lruvec)->node_zones[zid],
				       NR_ZONE_LRU_BASE + lru);
		lru_size -= min(size, lru_size);
	}

	return lru_size;
364
365
366

}

Linus Torvalds's avatar
Linus Torvalds committed
367
/*
Glauber Costa's avatar
Glauber Costa committed
368
 * Add a shrinker callback to be called from the vm.
Linus Torvalds's avatar
Linus Torvalds committed
369
 */
370
int prealloc_shrinker(struct shrinker *shrinker)
Linus Torvalds's avatar
Linus Torvalds committed
371
{
Glauber Costa's avatar
Glauber Costa committed
372
373
374
375
376
377
378
379
	size_t size = sizeof(*shrinker->nr_deferred);

	if (shrinker->flags & SHRINKER_NUMA_AWARE)
		size *= nr_node_ids;

	shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
	if (!shrinker->nr_deferred)
		return -ENOMEM;
380
381
382
383
384
385

	if (shrinker->flags & SHRINKER_MEMCG_AWARE) {
		if (prealloc_memcg_shrinker(shrinker))
			goto free_deferred;
	}

386
	return 0;
387
388
389
390
391

free_deferred:
	kfree(shrinker->nr_deferred);
	shrinker->nr_deferred = NULL;
	return -ENOMEM;
392
393
394
395
}

void free_prealloced_shrinker(struct shrinker *shrinker)
{
396
397
398
399
400
401
	if (!shrinker->nr_deferred)
		return;

	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
		unregister_memcg_shrinker(shrinker);

402
403
404
	kfree(shrinker->nr_deferred);
	shrinker->nr_deferred = NULL;
}
Glauber Costa's avatar
Glauber Costa committed
405

406
407
void register_shrinker_prepared(struct shrinker *shrinker)
{
408
409
	down_write(&shrinker_rwsem);
	list_add_tail(&shrinker->list, &shrinker_list);
410
#ifdef CONFIG_MEMCG_KMEM
411
412
	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
		idr_replace(&shrinker_idr, shrinker, shrinker->id);
413
#endif
414
	up_write(&shrinker_rwsem);
415
416
417
418
419
420
421
422
423
}

int register_shrinker(struct shrinker *shrinker)
{
	int err = prealloc_shrinker(shrinker);

	if (err)
		return err;
	register_shrinker_prepared(shrinker);
Glauber Costa's avatar
Glauber Costa committed
424
	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
425
}
426
EXPORT_SYMBOL(register_shrinker);
Linus Torvalds's avatar
Linus Torvalds committed
427
428
429
430

/*
 * Remove one
 */
431
void unregister_shrinker(struct shrinker *shrinker)
Linus Torvalds's avatar
Linus Torvalds committed
432
{
433
434
	if (!shrinker->nr_deferred)
		return;
435
436
	if (shrinker->flags & SHRINKER_MEMCG_AWARE)
		unregister_memcg_shrinker(shrinker);
Linus Torvalds's avatar
Linus Torvalds committed
437
438
439
	down_write(&shrinker_rwsem);
	list_del(&shrinker->list);
	up_write(&shrinker_rwsem);
440
	kfree(shrinker->nr_deferred);
441
	shrinker->nr_deferred = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
442
}
443
EXPORT_SYMBOL(unregister_shrinker);
Linus Torvalds's avatar
Linus Torvalds committed
444
445

#define SHRINK_BATCH 128
Glauber Costa's avatar
Glauber Costa committed
446

447
static unsigned long do_shrink_slab(struct shrink_control *shrinkctl,
448
				    struct shrinker *shrinker, int priority)
Glauber Costa's avatar
Glauber Costa committed
449
450
451
452
{
	unsigned long freed = 0;
	unsigned long long delta;
	long total_scan;
453
	long freeable;
Glauber Costa's avatar
Glauber Costa committed
454
455
456
457
458
	long nr;
	long new_nr;
	int nid = shrinkctl->nid;
	long batch_size = shrinker->batch ? shrinker->batch
					  : SHRINK_BATCH;
459
	long scanned = 0, next_deferred;
Glauber Costa's avatar
Glauber Costa committed
460

461
462
463
	if (!(shrinker->flags & SHRINKER_NUMA_AWARE))
		nid = 0;

464
	freeable = shrinker->count_objects(shrinker, shrinkctl);
465
466
	if (freeable == 0 || freeable == SHRINK_EMPTY)
		return freeable;
Glauber Costa's avatar
Glauber Costa committed
467
468
469
470
471
472
473
474
475

	/*
	 * copy the current shrinker scan count into a local variable
	 * and zero it so that other concurrent shrinker invocations
	 * don't also do this scanning work.
	 */
	nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);

	total_scan = nr;
476
477
478
	delta = freeable >> priority;
	delta *= 4;
	do_div(delta, shrinker->seeks);
479
480
481
482
483
484
485
486
487
488
489

	/*
	 * Make sure we apply some minimal pressure on default priority
	 * even on small cgroups. Stale objects are not only consuming memory
	 * by themselves, but can also hold a reference to a dying cgroup,
	 * preventing it from being reclaimed. A dying cgroup with all
	 * corresponding structures like per-cpu stats and kmem caches
	 * can be really big, so it may lead to a significant waste of memory.
	 */
	delta = max_t(unsigned long long, delta, min(freeable, batch_size));

Glauber Costa's avatar
Glauber Costa committed
490
491
	total_scan += delta;
	if (total_scan < 0) {
492
		pr_err("shrink_slab: %pF negative objects to delete nr=%ld\n",
493
		       shrinker->scan_objects, total_scan);
494
		total_scan = freeable;
495
496
497
		next_deferred = nr;
	} else
		next_deferred = total_scan;
Glauber Costa's avatar
Glauber Costa committed
498
499
500
501
502
503
504

	/*
	 * We need to avoid excessive windup on filesystem shrinkers
	 * due to large numbers of GFP_NOFS allocations causing the
	 * shrinkers to return -1 all the time. This results in a large
	 * nr being built up so when a shrink that can do some work
	 * comes along it empties the entire cache due to nr >>>
505
	 * freeable. This is bad for sustaining a working set in
Glauber Costa's avatar
Glauber Costa committed
506
507
508
509
510
	 * memory.
	 *
	 * Hence only allow the shrinker to scan the entire cache when
	 * a large delta change is calculated directly.
	 */
511
512
	if (delta < freeable / 4)
		total_scan = min(total_scan, freeable / 2);
Glauber Costa's avatar
Glauber Costa committed
513
514
515
516
517
518

	/*
	 * Avoid risking looping forever due to too large nr value:
	 * never try to free more than twice the estimate number of
	 * freeable entries.
	 */
519
520
	if (total_scan > freeable * 2)
		total_scan = freeable * 2;
Glauber Costa's avatar
Glauber Costa committed
521
522

	trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
523
				   freeable, delta, total_scan, priority);
Glauber Costa's avatar
Glauber Costa committed
524

525
526
527
528
529
530
531
532
533
534
535
	/*
	 * Normally, we should not scan less than batch_size objects in one
	 * pass to avoid too frequent shrinker calls, but if the slab has less
	 * than batch_size objects in total and we are really tight on memory,
	 * we will try to reclaim all available objects, otherwise we can end
	 * up failing allocations although there are plenty of reclaimable
	 * objects spread over several slabs with usage less than the
	 * batch_size.
	 *
	 * We detect the "tight on memory" situations by looking at the total
	 * number of objects we want to scan (total_scan). If it is greater
536
	 * than the total number of objects on slab (freeable), we must be
537
538
539
540
	 * scanning at high prio and therefore should try to reclaim as much as
	 * possible.
	 */
	while (total_scan >= batch_size ||
541
	       total_scan >= freeable) {
542
		unsigned long ret;
543
		unsigned long nr_to_scan = min(batch_size, total_scan);
Glauber Costa's avatar
Glauber Costa committed
544

545
		shrinkctl->nr_to_scan = nr_to_scan;
546
		shrinkctl->nr_scanned = nr_to_scan;
547
548
549
550
		ret = shrinker->scan_objects(shrinker, shrinkctl);
		if (ret == SHRINK_STOP)
			break;
		freed += ret;
Glauber Costa's avatar
Glauber Costa committed
551

552
553
554
		count_vm_events(SLABS_SCANNED, shrinkctl->nr_scanned);
		total_scan -= shrinkctl->nr_scanned;
		scanned += shrinkctl->nr_scanned;
Glauber Costa's avatar
Glauber Costa committed
555
556
557
558

		cond_resched();
	}

559
560
561
562
	if (next_deferred >= scanned)
		next_deferred -= scanned;
	else
		next_deferred = 0;
Glauber Costa's avatar
Glauber Costa committed
563
564
565
566
567
	/*
	 * move the unused scan count back into the shrinker in a
	 * manner that handles concurrent updates. If we exhausted the
	 * scan, there is no need to do an update.
	 */
568
569
	if (next_deferred > 0)
		new_nr = atomic_long_add_return(next_deferred,
Glauber Costa's avatar
Glauber Costa committed
570
571
572
573
						&shrinker->nr_deferred[nid]);
	else
		new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);

574
	trace_mm_shrink_slab_end(shrinker, nid, freed, nr, new_nr, total_scan);
Glauber Costa's avatar
Glauber Costa committed
575
	return freed;
576
577
}

578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
#ifdef CONFIG_MEMCG_KMEM
static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
			struct mem_cgroup *memcg, int priority)
{
	struct memcg_shrinker_map *map;
	unsigned long freed = 0;
	int ret, i;

	if (!memcg_kmem_enabled() || !mem_cgroup_online(memcg))
		return 0;

	if (!down_read_trylock(&shrinker_rwsem))
		return 0;

	map = rcu_dereference_protected(memcg->nodeinfo[nid]->shrinker_map,
					true);
	if (unlikely(!map))
		goto unlock;

	for_each_set_bit(i, map->map, shrinker_nr_max) {
		struct shrink_control sc = {
			.gfp_mask = gfp_mask,
			.nid = nid,
			.memcg = memcg,
		};
		struct shrinker *shrinker;

		shrinker = idr_find(&shrinker_idr, i);
606
607
608
		if (unlikely(!shrinker || shrinker == SHRINKER_REGISTERING)) {
			if (!shrinker)
				clear_bit(i, map->map);
609
610
611
612
			continue;
		}

		ret = do_shrink_slab(&sc, shrinker, priority);
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
		if (ret == SHRINK_EMPTY) {
			clear_bit(i, map->map);
			/*
			 * After the shrinker reported that it had no objects to
			 * free, but before we cleared the corresponding bit in
			 * the memcg shrinker map, a new object might have been
			 * added. To make sure, we have the bit set in this
			 * case, we invoke the shrinker one more time and reset
			 * the bit if it reports that it is not empty anymore.
			 * The memory barrier here pairs with the barrier in
			 * memcg_set_shrinker_bit():
			 *
			 * list_lru_add()     shrink_slab_memcg()
			 *   list_add_tail()    clear_bit()
			 *   <MB>               <MB>
			 *   set_bit()          do_shrink_slab()
			 */
			smp_mb__after_atomic();
			ret = do_shrink_slab(&sc, shrinker, priority);
			if (ret == SHRINK_EMPTY)
				ret = 0;
			else
				memcg_set_shrinker_bit(memcg, nid, i);
		}
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
		freed += ret;

		if (rwsem_is_contended(&shrinker_rwsem)) {
			freed = freed ? : 1;
			break;
		}
	}
unlock:
	up_read(&shrinker_rwsem);
	return freed;
}
#else /* CONFIG_MEMCG_KMEM */
static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
			struct mem_cgroup *memcg, int priority)
{
	return 0;
}
#endif /* CONFIG_MEMCG_KMEM */

656
/**
657
 * shrink_slab - shrink slab caches
658
659
 * @gfp_mask: allocation context
 * @nid: node whose slab caches to target
660
 * @memcg: memory cgroup whose slab caches to target
661
 * @priority: the reclaim priority
Linus Torvalds's avatar
Linus Torvalds committed
662
 *
663
 * Call the shrink functions to age shrinkable caches.
Linus Torvalds's avatar
Linus Torvalds committed
664
 *
665
666
 * @nid is passed along to shrinkers with SHRINKER_NUMA_AWARE set,
 * unaware shrinkers will receive a node id of 0 instead.
Linus Torvalds's avatar
Linus Torvalds committed
667
 *
668
669
 * @memcg specifies the memory cgroup to target. Unaware shrinkers
 * are called only if it is the root cgroup.
670
 *
671
672
 * @priority is sc->priority, we take the number of objects and >> by priority
 * in order to get the scan target.
673
 *
674
 * Returns the number of reclaimed slab objects.
Linus Torvalds's avatar
Linus Torvalds committed
675
 */
676
677
static unsigned long shrink_slab(gfp_t gfp_mask, int nid,
				 struct mem_cgroup *memcg,
678
				 int priority)
Linus Torvalds's avatar
Linus Torvalds committed
679
680
{
	struct shrinker *shrinker;
Dave Chinner's avatar
Dave Chinner committed
681
	unsigned long freed = 0;
682
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
683

684
	if (!mem_cgroup_is_root(memcg))
685
		return shrink_slab_memcg(gfp_mask, nid, memcg, priority);
686

687
	if (!down_read_trylock(&shrinker_rwsem))
688
		goto out;
Linus Torvalds's avatar
Linus Torvalds committed
689
690

	list_for_each_entry(shrinker, &shrinker_list, list) {
691
692
693
		struct shrink_control sc = {
			.gfp_mask = gfp_mask,
			.nid = nid,
694
			.memcg = memcg,
695
		};
696

697
698
699
700
		ret = do_shrink_slab(&sc, shrinker, priority);
		if (ret == SHRINK_EMPTY)
			ret = 0;
		freed += ret;
701
702
703
704
705
706
707
708
709
		/*
		 * Bail out if someone want to register a new shrinker to
		 * prevent the regsitration from being stalled for long periods
		 * by parallel ongoing shrinking.
		 */
		if (rwsem_is_contended(&shrinker_rwsem)) {
			freed = freed ? : 1;
			break;
		}
Linus Torvalds's avatar
Linus Torvalds committed
710
	}
711

Linus Torvalds's avatar
Linus Torvalds committed
712
	up_read(&shrinker_rwsem);
713
714
out:
	cond_resched();
Dave Chinner's avatar
Dave Chinner committed
715
	return freed;
Linus Torvalds's avatar
Linus Torvalds committed
716
717
}

718
719
720
721
722
723
724
725
void drop_slab_node(int nid)
{
	unsigned long freed;

	do {
		struct mem_cgroup *memcg = NULL;

		freed = 0;
726
		memcg = mem_cgroup_iter(NULL, NULL, NULL);
727
		do {
728
			freed += shrink_slab(GFP_KERNEL, nid, memcg, 0);
729
730
731
732
733
734
735
736
737
738
739
740
		} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
	} while (freed > 10);
}

void drop_slab(void)
{
	int nid;

	for_each_online_node(nid)
		drop_slab_node(nid);
}

Linus Torvalds's avatar
Linus Torvalds committed
741
742
static inline int is_page_cache_freeable(struct page *page)
{
743
744
745
746
747
	/*
	 * A freeable page cache page is referenced only by the caller
	 * that isolated the page, the page cache radix tree and
	 * optional buffer heads at page->private.
	 */
748
749
750
	int radix_pins = PageTransHuge(page) && PageSwapCache(page) ?
		HPAGE_PMD_NR : 1;
	return page_count(page) - page_has_private(page) == 1 + radix_pins;
Linus Torvalds's avatar
Linus Torvalds committed
751
752
}

753
static int may_write_to_inode(struct inode *inode, struct scan_control *sc)
Linus Torvalds's avatar
Linus Torvalds committed
754
{
755
	if (current->flags & PF_SWAPWRITE)
Linus Torvalds's avatar
Linus Torvalds committed
756
		return 1;
757
	if (!inode_write_congested(inode))
Linus Torvalds's avatar
Linus Torvalds committed
758
		return 1;
759
	if (inode_to_bdi(inode) == current->backing_dev_info)
Linus Torvalds's avatar
Linus Torvalds committed
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
		return 1;
	return 0;
}

/*
 * We detected a synchronous write error writing a page out.  Probably
 * -ENOSPC.  We need to propagate that into the address_space for a subsequent
 * fsync(), msync() or close().
 *
 * The tricky part is that after writepage we cannot touch the mapping: nothing
 * prevents it from being freed up.  But we have a ref on the page and once
 * that page is locked, the mapping is pinned.
 *
 * We're allowed to run sleeping lock_page() here because we know the caller has
 * __GFP_FS.
 */
static void handle_write_error(struct address_space *mapping,
				struct page *page, int error)
{
Jens Axboe's avatar
Jens Axboe committed
779
	lock_page(page);
780
781
	if (page_mapping(page) == mapping)
		mapping_set_error(mapping, error);
Linus Torvalds's avatar
Linus Torvalds committed
782
783
784
	unlock_page(page);
}

785
786
787
788
789
790
791
792
793
794
795
796
/* possible outcome of pageout() */
typedef enum {
	/* failed to write page out, page is locked */
	PAGE_KEEP,
	/* move page to the active list, page is locked */
	PAGE_ACTIVATE,
	/* page has been sent to the disk successfully, page is unlocked */
	PAGE_SUCCESS,
	/* page is clean and locked */
	PAGE_CLEAN,
} pageout_t;

Linus Torvalds's avatar
Linus Torvalds committed
797
/*
798
799
 * pageout is called by shrink_page_list() for each dirty page.
 * Calls ->writepage().
Linus Torvalds's avatar
Linus Torvalds committed
800
 */
801
static pageout_t pageout(struct page *page, struct address_space *mapping,
802
			 struct scan_control *sc)
Linus Torvalds's avatar
Linus Torvalds committed
803
804
805
806
807
808
809
810
{
	/*
	 * If the page is dirty, only perform writeback if that write
	 * will be non-blocking.  To prevent this allocation from being
	 * stalled by pagecache activity.  But note that there may be
	 * stalls if we need to run get_block().  We could test
	 * PagePrivate for that.
	 *
811
	 * If this process is currently in __generic_file_write_iter() against
Linus Torvalds's avatar
Linus Torvalds committed
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
	 * this page's queue, we can perform writeback even if that
	 * will block.
	 *
	 * If the page is swapcache, write it back even if that would
	 * block, for some throttling. This happens by accident, because
	 * swap_backing_dev_info is bust: it doesn't reflect the
	 * congestion state of the swapdevs.  Easy to fix, if needed.
	 */
	if (!is_page_cache_freeable(page))
		return PAGE_KEEP;
	if (!mapping) {
		/*
		 * Some data journaling orphaned pages can have
		 * page->mapping == NULL while being dirty with clean buffers.
		 */
827
		if (page_has_private(page)) {
Linus Torvalds's avatar
Linus Torvalds committed
828
829
			if (try_to_free_buffers(page)) {
				ClearPageDirty(page);
830
				pr_info("%s: orphaned page\n", __func__);
Linus Torvalds's avatar
Linus Torvalds committed
831
832
833
834
835
836
837
				return PAGE_CLEAN;
			}
		}
		return PAGE_KEEP;
	}
	if (mapping->a_ops->writepage == NULL)
		return PAGE_ACTIVATE;
838
	if (!may_write_to_inode(mapping->host, sc))
Linus Torvalds's avatar
Linus Torvalds committed
839
840
841
842
843
844
845
		return PAGE_KEEP;

	if (clear_page_dirty_for_io(page)) {
		int res;
		struct writeback_control wbc = {
			.sync_mode = WB_SYNC_NONE,
			.nr_to_write = SWAP_CLUSTER_MAX,
846
847
			.range_start = 0,
			.range_end = LLONG_MAX,
Linus Torvalds's avatar
Linus Torvalds committed
848
849
850
851
852
853
854
			.for_reclaim = 1,
		};

		SetPageReclaim(page);
		res = mapping->a_ops->writepage(page, &wbc);
		if (res < 0)
			handle_write_error(mapping, page, res);
855
		if (res == AOP_WRITEPAGE_ACTIVATE) {
Linus Torvalds's avatar
Linus Torvalds committed
856
857
858
			ClearPageReclaim(page);
			return PAGE_ACTIVATE;
		}
859

Linus Torvalds's avatar
Linus Torvalds committed
860
861
862
863
		if (!PageWriteback(page)) {
			/* synchronous write or broken a_ops? */
			ClearPageReclaim(page);
		}
864
		trace_mm_vmscan_writepage(page);
865
		inc_node_page_state(page, NR_VMSCAN_WRITE);
Linus Torvalds's avatar
Linus Torvalds committed
866
867
868
869
870
871
		return PAGE_SUCCESS;
	}

	return PAGE_CLEAN;
}

872
/*
Nick Piggin's avatar
Nick Piggin committed
873
874
 * Same as remove_mapping, but if the page is removed from the mapping, it
 * gets returned with a refcount of 0.
875
 */
876
877
static int __remove_mapping(struct address_space *mapping, struct page *page,
			    bool reclaimed)
878
{
879
	unsigned long flags;
880
	int refcount;
881

882
883
	BUG_ON(!PageLocked(page));
	BUG_ON(mapping != page_mapping(page));
884

Matthew Wilcox's avatar
Matthew Wilcox committed
885
	xa_lock_irqsave(&mapping->i_pages, flags);
886
	/*
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
	 * The non racy check for a busy page.
	 *
	 * Must be careful with the order of the tests. When someone has
	 * a ref to the page, it may be possible that they dirty it then
	 * drop the reference. So if PageDirty is tested before page_count
	 * here, then the following race may occur:
	 *
	 * get_user_pages(&page);
	 * [user mapping goes away]
	 * write_to(page);
	 *				!PageDirty(page)    [good]
	 * SetPageDirty(page);
	 * put_page(page);
	 *				!page_count(page)   [good, discard it]
	 *
	 * [oops, our write_to data is lost]
	 *
	 * Reversing the order of the tests ensures such a situation cannot
	 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
906
	 * load is not satisfied before that of page->_refcount.
907
908
	 *
	 * Note that if SetPageDirty is always performed via set_page_dirty,
Matthew Wilcox's avatar
Matthew Wilcox committed
909
	 * and thus under the i_pages lock, then this ordering is not required.
910
	 */
911
912
913
914
915
	if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
		refcount = 1 + HPAGE_PMD_NR;
	else
		refcount = 2;
	if (!page_ref_freeze(page, refcount))
916
		goto cannot_free;
917
	/* note: atomic_cmpxchg in page_ref_freeze provides the smp_rmb */
Nick Piggin's avatar
Nick Piggin committed
918
	if (unlikely(PageDirty(page))) {
919
		page_ref_unfreeze(page, refcount);
920
		goto cannot_free;
Nick Piggin's avatar
Nick Piggin committed
921
	}
922
923
924

	if (PageSwapCache(page)) {
		swp_entry_t swap = { .val = page_private(page) };
925
		mem_cgroup_swapout(page, swap);
926
		__delete_from_swap_cache(page);
Matthew Wilcox's avatar
Matthew Wilcox committed
927
		xa_unlock_irqrestore(&mapping->i_pages, flags);
928
		put_swap_page(page, swap);
Nick Piggin's avatar
Nick Piggin committed
929
	} else {
930
		void (*freepage)(struct page *);
931
		void *shadow = NULL;
932
933

		freepage = mapping->a_ops->freepage;
934
935
936
937
938
939
940
941
942
		/*
		 * Remember a shadow entry for reclaimed file cache in
		 * order to detect refaults, thus thrashing, later on.
		 *
		 * But don't store shadows in an address space that is
		 * already exiting.  This is not just an optizimation,
		 * inode reclaim needs to empty out the radix tree or
		 * the nodes are lost.  Don't plant shadows behind its
		 * back.
943
944
945
946
947
		 *
		 * We also don't store shadows for DAX mappings because the
		 * only page cache pages found in these are zero pages
		 * covering holes, and because we don't want to mix DAX
		 * exceptional entries and shadow exceptional entries in the
Matthew Wilcox's avatar
Matthew Wilcox committed
948
		 * same address_space.
949
950
		 */
		if (reclaimed && page_is_file_cache(page) &&
951
		    !mapping_exiting(mapping) && !dax_mapping(mapping))
952
			shadow = workingset_eviction(mapping, page);
953
		__delete_from_page_cache(page, shadow);
Matthew Wilcox's avatar
Matthew Wilcox committed
954
		xa_unlock_irqrestore(&mapping->i_pages, flags);
955
956
957

		if (freepage != NULL)
			freepage(page);
958
959
960
961
962
	}

	return 1;

cannot_free:
Matthew Wilcox's avatar
Matthew Wilcox committed
963
	xa_unlock_irqrestore(&mapping->i_pages, flags);
964
965
966
	return 0;
}

Nick Piggin's avatar
Nick Piggin committed
967
968
969
970
971
972
973
974
/*
 * Attempt to detach a locked page from its ->mapping.  If it is dirty or if
 * someone else has a ref on the page, abort and return 0.  If it was
 * successfully detached, return 1.  Assumes the caller has a single ref on
 * this page.
 */
int remove_mapping(struct address_space *mapping, struct page *page)
{
975
	if (__remove_mapping(mapping, page, false)) {
Nick Piggin's avatar
Nick Piggin committed
976
977
978
979
980
		/*
		 * Unfreezing the refcount with 1 rather than 2 effectively
		 * drops the pagecache ref for us without requiring another
		 * atomic operation.
		 */
981
		page_ref_unfreeze(page, 1);
Nick Piggin's avatar
Nick Piggin committed
982
983
984
985
986
		return 1;
	}
	return 0;
}

987
988
989
990
991
992
993
994
995
996
997
/**
 * putback_lru_page - put previously isolated page onto appropriate LRU list
 * @page: page to be put back to appropriate lru list
 *
 * Add previously isolated @page to appropriate LRU list.
 * Page may still be unevictable for other reasons.
 *
 * lru_lock must not be held, interrupts must be enabled.
 */
void putback_lru_page(struct page *page)
{
998
	lru_cache_add(page);
999
1000
1001
	put_page(page);		/* drop ref from isolate */
}

1002
1003
1004
enum page_references {
	PAGEREF_RECLAIM,
	PAGEREF_RECLAIM_CLEAN,
1005
	PAGEREF_KEEP,
1006
1007
1008
1009
1010
1011
	PAGEREF_ACTIVATE,
};

static enum page_references page_check_references(struct page *page,
						  struct scan_control *sc)
{
1012
	int referenced_ptes, referenced_page;
1013
1014
	unsigned long vm_flags;

1015
1016
	referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
					  &vm_flags);
1017
	referenced_page = TestClearPageReferenced(page);
1018
1019
1020
1021
1022
1023
1024
1025

	/*
	 * Mlock lost the isolation race with us.  Let try_to_unmap()
	 * move the page to the unevictable list.
	 */
	if (vm_flags & VM_LOCKED)
		return PAGEREF_RECLAIM;

1026
	if (referenced_ptes) {
1027
		if (PageSwapBacked(page))
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
			return PAGEREF_ACTIVATE;
		/*
		 * All mapped pages start out with page table
		 * references from the instantiating fault, so we need
		 * to look twice if a mapped file page is used more
		 * than once.
		 *
		 * Mark it and spare it for another trip around the
		 * inactive list.  Another page table reference will
		 * lead to its activation.
		 *
		 * Note: the mark is set for activated pages as well
		 * so that recently deactivated but used pages are
		 * quickly recovered.
		 */
		SetPageReferenced(page);

1045
		if (referenced_page || referenced_ptes > 1)
1046
1047
			return PAGEREF_ACTIVATE;

1048
1049
1050
1051
1052
1053
		/*
		 * Activate file-backed executable pages after first usage.
		 */
		if (vm_flags & VM_EXEC)
			return PAGEREF_ACTIVATE;

1054
1055
		return PAGEREF_KEEP;
	}
1056
1057

	/* Reclaim if clean, defer dirty pages to writeback */
1058
	if (referenced_page && !PageSwapBacked(page))
1059
1060
1061
		return PAGEREF_RECLAIM_CLEAN;

	return PAGEREF_RECLAIM;
1062
1063
}

1064
1065
1066
1067
/* Check if a page is dirty or under writeback */
static void page_check_dirty_writeback(struct page *page,
				       bool *dirty, bool *writeback)
{
1068
1069
	struct address_space *mapping;

1070
1071
1072
1073
	/*
	 * Anonymous pages are not handled by flushers and must be written
	 * from reclaim context. Do not stall reclaim based on them
	 */
Shaohua Li's avatar
Shaohua Li committed
1074
1075
	if (!page_is_file_cache(page) ||
	    (PageAnon(page) && !PageSwapBacked(page))) {
1076
1077
1078
1079
1080
1081
1082
1083
		*dirty = false;
		*writeback = false;
		return;
	}

	/* By default assume that the page flags are accurate */
	*dirty = PageDirty(page);
	*writeback = PageWriteback(page);
1084
1085
1086
1087
1088
1089
1090
1091

	/* Verify dirty/writeback state if the filesystem supports it */
	if (!page_has_private(page))
		return;

	mapping = page_mapping(page);
	if (mapping && mapping->a_ops->is_dirty_writeback)
		mapping->a_ops->is_dirty_writeback(page, dirty, writeback);
1092
1093
}

Linus Torvalds's avatar
Linus Torvalds committed
1094
/*
1095
 * shrink_page_list() returns the number of reclaimed pages
Linus Torvalds's avatar
Linus Torvalds committed
1096
 */
1097
static unsigned long shrink_page_list(struct list_head *page_list,
1098
				      struct pglist_data *pgdat,
1099
				      struct scan_control *sc,
1100
				      enum ttu_flags ttu_flags,
1101
				      struct reclaim_stat *stat,
1102
				      bool force_reclaim)
Linus Torvalds's avatar
Linus Torvalds committed
1103
1104
{
	LIST_HEAD(ret_pages);
1105
	LIST_HEAD(free_pages);
Linus Torvalds's avatar
Linus Torvalds committed
1106
	int pgactivate = 0;
1107
1108
1109
1110
1111
1112
	unsigned nr_unqueued_dirty = 0;
	unsigned nr_dirty = 0;
	unsigned nr_congested = 0;
	unsigned nr_reclaimed = 0;
	unsigned nr_writeback = 0;
	unsigned nr_immediate = 0;
1113
1114
	unsigned nr_ref_keep = 0;
	unsigned nr_unmap_fail = 0;
Linus Torvalds's avatar
Linus Torvalds committed
1115
1116
1117
1118
1119
1120
1121

	cond_resched();

	while (!list_empty(page_list)) {
		struct address_space *mapping;
		struct page *page;
		int may_enter_fs;
1122
		enum page_references references = PAGEREF_RECLAIM_CLEAN;
1123
		bool dirty, writeback;
Linus Torvalds's avatar
Linus Torvalds committed
1124
1125
1126
1127
1128
1129

		cond_resched();

		page = lru_to_page(page_list);
		list_del(&page->lru);

Nick Piggin's avatar
Nick Piggin committed
1130
		if (!trylock_page(page))
Linus Torvalds's avatar
Linus Torvalds committed
1131
1132
			goto keep;

1133
		VM_BUG_ON_PAGE(PageActive(page), page);
Linus Torvalds's avatar
Linus Torvalds committed
1134
1135

		sc->nr_scanned++;
1136

1137
		if (unlikely(!page_evictable(page)))
Minchan Kim's avatar
Minchan Kim committed
1138
			goto activate_locked;
1139

1140
		if (!sc->may_unmap && page_mapped(page))
1141
1142
			goto keep_locked;

Linus Torvalds's avatar
Linus Torvalds committed
1143
		/* Double the slab pressure for mapped and swapcache pages */
Shaohua Li's avatar
Shaohua Li committed
1144
1145
		if ((page_mapped(page) || PageSwapCache(page)) &&
		    !(PageAnon(page) && !PageSwapBacked(page)))
Linus Torvalds's avatar
Linus Torvalds committed
1146
1147
			sc->nr_scanned++;

1148
1149
1150
		may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
			(PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));

1151
		/*
1152
		 * The number of dirty pages determines if a node is marked
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
		 * reclaim_congested which affects wait_iff_congested. kswapd
		 * will stall and start writing pages if the tail of the LRU
		 * is all dirty unqueued pages.
		 */
		page_check_dirty_writeback(page, &dirty, &writeback);
		if (dirty || writeback)
			nr_dirty++;

		if (dirty && !writeback)
			nr_unqueued_dirty++;

1164
1165
1166
1167
1168
1169
		/*
		 * Treat this page as congested if the underlying BDI is or if
		 * pages are cycling through the LRU so quickly that the
		 * pages marked for immediate reclaim are making it to the
		 * end of the LRU a second time.
		 */
1170
		mapping = page_mapping(page);
1171
		if (((dirty || writeback) && mapping &&
1172
		     inode_write_congested(mapping->host)) ||
1173
		    (writeback && PageReclaim(page)))
1174
1175
			nr_congested++;

1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
		/*
		 * If a page at the tail of the LRU is under writeback, there
		 * are three cases to consider.
		 *
		 * 1) If reclaim is encountering an excessive number of pages
		 *    under writeback and this page is both under writeback and
		 *    PageReclaim then it indicates that pages are being queued
		 *    for IO but are being recycled through the LRU before the
		 *    IO can complete. Waiting on the page itself risks an
		 *    indefinite stall if it is impossible to writeback the
		 *    page due to IO error or disconnected storage so instead
1187
1188
		 *    note that the LRU is being scanned too quickly and the
		 *    caller can stall after page list has been processed.
1189
		 *
1190
		 * 2) Global or new memcg reclaim encounters a page that is
1191
1192
1193
		 *    not marked for immediate reclaim, or the caller does not
		 *    have __GFP_FS (or __GFP_IO if it's simply going to swap,
		 *    not to fs). In this case mark the page for immediate
1194
		 *    reclaim and continue scanning.
1195
		 *
1196
1197
		 *    Require may_enter_fs because we would wait on fs, which
		 *    may not have submitted IO yet. And the loop driver might
1198
1199
1200
1201
1202
		 *    enter reclaim, and deadlock if it waits on a page for
		 *    which it is needed to do the write (loop masks off
		 *    __GFP_IO|__GFP_FS for this reason); but more thought
		 *    would probably show more reasons.
		 *
1203
		 * 3) Legacy memcg encounters a page that is already marked
1204
1205
1206
1207
		 *    PageReclaim. memcg does not have any dirty pages
		 *    throttling so we could easily OOM just because too many
		 *    pages are in writeback and there is nothing else to
		 *    reclaim. Wait for the writeback to complete.
1208
1209
1210
1211
1212
1213
1214
1215
1216
		 *
		 * In cases 1) and 2) we activate the pages to get them out of
		 * the way while we continue scanning for clean pages on the
		 * inactive list and refilling from the active list. The
		 * observation here is that waiting for disk writes is more
		 * expensive than potentially causing reloads down the line.
		 * Since they're marked for immediate reclaim, they won't put
		 * memory pressure on the cache working set any longer than it
		 * takes to write them to disk.
1217
		 */
1218
		if (PageWriteback(page)) {
1219
1220
1221
			/* Case 1 above */
			if (current_is_kswapd() &&
			    PageReclaim(page) &&
1222
			    test_bit(PGDAT_WRITEBACK, &pgdat->flags)) {
1223
				nr_immediate++;
1224
				goto activate_locked;
1225
1226

			/* Case 2 above */
1227
			} else if (sane_reclaim(sc) ||
1228
			    !PageReclaim(page) || !may_enter_fs) {
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
				/*
				 * This is slightly racy - end_page_writeback()
				 * might have just cleared PageReclaim, then
				 * setting PageReclaim here end up interpreted
				 * as PageReadahead - but that does not matter
				 * enough to care.  What we do want is for this
				 * page to have PageReclaim set next time memcg
				 * reclaim reaches the tests above, so it will
				 * then wait_on_page_writeback() to avoid OOM;
				 * and it's also appropriate in global reclaim.
				 */
				SetPageReclaim(page);
1241
				nr_writeback++;
1242
				goto activate_locked;
1243
1244
1245

			/* Case 3 above */
			} else {
1246
				unlock_page(page);
1247
				wait_on_page_writeback(page);
1248
1249
1250
				/* then go back and try same page again */
				list_add_tail(&page->lru, page_list);
				continue;
1251
			}
1252
		}
Linus Torvalds's avatar
Linus Torvalds committed
1253

1254
1255
1256
		if (!force_reclaim)
			references = page_check_references(page, sc);

1257
1258
		switch (references) {
		case PAGEREF_ACTIVATE:
Linus Torvalds's avatar
Linus Torvalds committed
1259
			goto activate_locked;
1260
		case PAGEREF_KEEP:
1261
			nr_ref_keep++;
1262
			goto keep_locked;
1263
1264
1265
1266
		case PAGEREF_RECLAIM:
		case PAGEREF_RECLAIM_CLEAN:
			; /* try to reclaim the page below */
		}
Linus Torvalds's avatar
Linus Torvalds committed
1267
1268
1269
1270

		/*
		 * Anonymous process memory has backing store?
		 * Try to allocate it some swap space here.
Shaohua Li's avatar
Shaohua Li committed
1271
		 * Lazyfree page could be freed directly
Linus Torvalds's avatar
Linus Torvalds committed
1272
		 */
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
		if (PageAnon(page) && PageSwapBacked(page)) {
			if (!PageSwapCache(page)) {
				if (!(sc->gfp_mask & __GFP_IO))
					goto keep_locked;
				if (PageTransHuge(page)) {
					/* cannot split THP, skip it */
					if (!can_split_huge_page(page, NULL))
						goto activate_locked;
					/*
					 * Split pages without a PMD map right
					 * away. Chances are some or all of the
					 * tail pages can be freed without IO.
					 */
					if (!compound_mapcount(page) &&
					    split_huge_page_to_list(page,
								    page_list))
						goto activate_locked;
				}
				if (!add_to_swap(page)) {
					if (!PageTransHuge(page))
						goto activate_locked;
					/* Fallback to swap normal pages */
					if (split_huge_page_to_list(page,
								    page_list))
						goto activate_locked;
1298
1299
1300
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
					count_vm_event(THP_SWPOUT_FALLBACK);
#endif
1301
1302
1303
					if (!add_to_swap(page))
						goto activate_locked;
				}
1304

1305
				may_enter_fs = 1;
Linus Torvalds's avatar
Linus Torvalds committed
1306

1307
1308
1309
				/* Adding to swap updated mapping */
				mapping = page_mapping(page);
			}
1310
1311
1312
1313
		} else if (unlikely(PageTransHuge(page))) {
			/* Split file THP */
			if (split_huge_page_to_list(page, page_list))
				goto keep_locked;
1314
		}
Linus Torvalds's avatar
Linus Torvalds committed
1315
1316
1317
1318
1319

		/*
		 * The page is mapped into the page tables of one or more
		 * processes. Try to unmap it here.
		 */
Shaohua Li's avatar
Shaohua Li committed
1320
		if (page_mapped(page)) {
1321
1322
1323
1324
1325
			enum ttu_flags flags = ttu_flags | TTU_BATCH_FLUSH;

			if (unlikely(PageTransHuge(page)))
				flags |= TTU_SPLIT_HUGE_PMD;
			if (!try_to_unmap(page, flags)) {