mmzone.h 39.6 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
#ifndef _LINUX_MMZONE_H
#define _LINUX_MMZONE_H

#ifndef __ASSEMBLY__
5
#ifndef __GENERATING_BOUNDS_H
Linus Torvalds's avatar
Linus Torvalds committed
6
7
8
9

#include <linux/spinlock.h>
#include <linux/list.h>
#include <linux/wait.h>
10
#include <linux/bitops.h>
Linus Torvalds's avatar
Linus Torvalds committed
11
12
13
14
#include <linux/cache.h>
#include <linux/threads.h>
#include <linux/numa.h>
#include <linux/init.h>
15
#include <linux/seqlock.h>
16
#include <linux/nodemask.h>
17
#include <linux/pageblock-flags.h>
18
#include <linux/page-flags-layout.h>
Arun Sharma's avatar
Arun Sharma committed
19
#include <linux/atomic.h>
Ralf Baechle's avatar
Ralf Baechle committed
20
#include <asm/page.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
22
23
24
25
26
27

/* Free memory management - zoned buddy allocator.  */
#ifndef CONFIG_FORCE_MAX_ZONEORDER
#define MAX_ORDER 11
#else
#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
#endif
28
#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
Linus Torvalds's avatar
Linus Torvalds committed
29

Andy Whitcroft's avatar
Andy Whitcroft committed
30
31
32
/*
 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
 * costly to service.  That is between allocation orders which should
33
 * coalesce naturally under reasonable reclaim pressure and those which
Andy Whitcroft's avatar
Andy Whitcroft committed
34
35
36
37
 * will not.
 */
#define PAGE_ALLOC_COSTLY_ORDER 3

38
enum migratetype {
39
40
	MIGRATE_UNMOVABLE,
	MIGRATE_MOVABLE,
41
	MIGRATE_RECLAIMABLE,
42
43
	MIGRATE_PCPTYPES,	/* the number of types on the pcp lists */
	MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES,
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#ifdef CONFIG_CMA
	/*
	 * MIGRATE_CMA migration type is designed to mimic the way
	 * ZONE_MOVABLE works.  Only movable pages can be allocated
	 * from MIGRATE_CMA pageblocks and page allocator never
	 * implicitly change migration type of MIGRATE_CMA pageblock.
	 *
	 * The way to use it is to change migratetype of a range of
	 * pageblocks to MIGRATE_CMA which can be done by
	 * __free_pageblock_cma() function.  What is important though
	 * is that a range of pageblocks must be aligned to
	 * MAX_ORDER_NR_PAGES should biggest page be bigger then
	 * a single pageblock.
	 */
	MIGRATE_CMA,
#endif
60
#ifdef CONFIG_MEMORY_ISOLATION
61
	MIGRATE_ISOLATE,	/* can't allocate from here */
62
#endif
63
64
65
	MIGRATE_TYPES
};

66
67
68
/* In mm/page_alloc.c; keep in sync also with show_migration_types() there */
extern char * const migratetype_names[MIGRATE_TYPES];

69
70
#ifdef CONFIG_CMA
#  define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA)
Laura Abbott's avatar
Laura Abbott committed
71
#  define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA)
72
73
#else
#  define is_migrate_cma(migratetype) false
Laura Abbott's avatar
Laura Abbott committed
74
#  define is_migrate_cma_page(_page) false
75
#endif
76

77
78
79
80
81
static inline bool is_migrate_movable(int mt)
{
	return is_migrate_cma(mt) || mt == MIGRATE_MOVABLE;
}

82
83
84
85
#define for_each_migratetype_order(order, type) \
	for (order = 0; order < MAX_ORDER; order++) \
		for (type = 0; type < MIGRATE_TYPES; type++)

86
87
extern int page_group_by_mobility_disabled;

88
89
90
#define NR_MIGRATETYPE_BITS (PB_migrate_end - PB_migrate + 1)
#define MIGRATETYPE_MASK ((1UL << NR_MIGRATETYPE_BITS) - 1)

91
92
93
94
#define get_pageblock_migratetype(page)					\
	get_pfnblock_flags_mask(page, page_to_pfn(page),		\
			PB_migrate_end, MIGRATETYPE_MASK)

Linus Torvalds's avatar
Linus Torvalds committed
95
struct free_area {
96
	struct list_head	free_list[MIGRATE_TYPES];
Linus Torvalds's avatar
Linus Torvalds committed
97
98
99
100
101
102
	unsigned long		nr_free;
};

struct pglist_data;

/*
103
 * zone->lock and the zone lru_lock are two of the hottest locks in the kernel.
Linus Torvalds's avatar
Linus Torvalds committed
104
105
106
107
108
109
110
 * So add a wild amount of padding here to ensure that they fall into separate
 * cachelines.  There are very few zone structures in the machine, so space
 * consumption is not a concern here.
 */
#if defined(CONFIG_SMP)
struct zone_padding {
	char x[0];
111
} ____cacheline_internodealigned_in_smp;
Linus Torvalds's avatar
Linus Torvalds committed
112
113
114
115
116
#define ZONE_PADDING(name)	struct zone_padding name;
#else
#define ZONE_PADDING(name)
#endif

117
118
119
120
121
122
123
124
125
126
127
128
129
130
#ifdef CONFIG_NUMA
enum numa_stat_item {
	NUMA_HIT,		/* allocated in intended node */
	NUMA_MISS,		/* allocated in non intended node */
	NUMA_FOREIGN,		/* was intended here, hit elsewhere */
	NUMA_INTERLEAVE_HIT,	/* interleaver preferred this zone */
	NUMA_LOCAL,		/* allocation from local node */
	NUMA_OTHER,		/* allocation from other node */
	NR_VM_NUMA_STAT_ITEMS
};
#else
#define NR_VM_NUMA_STAT_ITEMS 0
#endif

131
enum zone_stat_item {
132
	/* First 128 byte cacheline (assuming 64 bit words) */
133
	NR_FREE_PAGES,
Minchan Kim's avatar
Minchan Kim committed
134
135
136
137
138
139
	NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */
	NR_ZONE_INACTIVE_ANON = NR_ZONE_LRU_BASE,
	NR_ZONE_ACTIVE_ANON,
	NR_ZONE_INACTIVE_FILE,
	NR_ZONE_ACTIVE_FILE,
	NR_ZONE_UNEVICTABLE,
140
	NR_ZONE_WRITE_PENDING,	/* Count of dirty, writeback and unstable pages */
Nick Piggin's avatar
Nick Piggin committed
141
	NR_MLOCK,		/* mlock()ed pages found and moved off LRU */
142
	NR_PAGETABLE,		/* used for pagetables */
143
	NR_KERNEL_STACK_KB,	/* measured in KiB */
144
	/* Second 128 byte cacheline */
145
	NR_BOUNCE,
Minchan Kim's avatar
Minchan Kim committed
146
147
#if IS_ENABLED(CONFIG_ZSMALLOC)
	NR_ZSPAGES,		/* allocated in zsmalloc */
148
#endif
149
	NR_FREE_CMA_PAGES,
150
151
	NR_VM_ZONE_STAT_ITEMS };

152
enum node_stat_item {
153
154
155
156
157
158
	NR_LRU_BASE,
	NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
	NR_ACTIVE_ANON,		/*  "     "     "   "       "         */
	NR_INACTIVE_FILE,	/*  "     "     "   "       "         */
	NR_ACTIVE_FILE,		/*  "     "     "   "       "         */
	NR_UNEVICTABLE,		/*  "     "     "   "       "         */
159
160
	NR_SLAB_RECLAIMABLE,
	NR_SLAB_UNRECLAIMABLE,
161
162
	NR_ISOLATED_ANON,	/* Temporary isolated pages from anon lru */
	NR_ISOLATED_FILE,	/* Temporary isolated pages from file lru */
163
164
165
	WORKINGSET_REFAULT,
	WORKINGSET_ACTIVATE,
	WORKINGSET_NODERECLAIM,
166
	NR_ANON_MAPPED,	/* Mapped anonymous pages */
167
168
	NR_FILE_MAPPED,	/* pagecache pages mapped into pagetables.
			   only modified from process context */
169
170
171
172
173
174
175
176
177
	NR_FILE_PAGES,
	NR_FILE_DIRTY,
	NR_WRITEBACK,
	NR_WRITEBACK_TEMP,	/* Writeback using temporary buffers */
	NR_SHMEM,		/* shmem pages (included tmpfs/GEM pages) */
	NR_SHMEM_THPS,
	NR_SHMEM_PMDMAPPED,
	NR_ANON_THPS,
	NR_UNSTABLE_NFS,	/* NFS unstable pages */
178
179
180
181
	NR_VMSCAN_WRITE,
	NR_VMSCAN_IMMEDIATE,	/* Prioritise for reclaim when writeback ends */
	NR_DIRTIED,		/* page dirtyings since bootup */
	NR_WRITTEN,		/* page writings since bootup */
182
183
184
	NR_VM_NODE_STAT_ITEMS
};

185
186
187
188
189
190
191
192
193
194
195
196
197
/*
 * We do arithmetic on the LRU lists in various places in the code,
 * so it is important to keep the active lists LRU_ACTIVE higher in
 * the array than the corresponding inactive lists, and to keep
 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
 *
 * This has to be kept in sync with the statistics in zone_stat_item
 * above and the descriptions in vmstat_text in mm/vmstat.c
 */
#define LRU_BASE 0
#define LRU_ACTIVE 1
#define LRU_FILE 2

198
enum lru_list {
199
200
201
202
	LRU_INACTIVE_ANON = LRU_BASE,
	LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
	LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
	LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
203
204
205
	LRU_UNEVICTABLE,
	NR_LRU_LISTS
};
206

Hugh Dickins's avatar
Hugh Dickins committed
207
#define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
208

Hugh Dickins's avatar
Hugh Dickins committed
209
#define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
210

Hugh Dickins's avatar
Hugh Dickins committed
211
static inline int is_file_lru(enum lru_list lru)
212
{
Hugh Dickins's avatar
Hugh Dickins committed
213
	return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
214
215
}

Hugh Dickins's avatar
Hugh Dickins committed
216
static inline int is_active_lru(enum lru_list lru)
217
{
Hugh Dickins's avatar
Hugh Dickins committed
218
	return (lru == LRU_ACTIVE_ANON || lru == LRU_ACTIVE_FILE);
219
220
}

221
222
223
struct zone_reclaim_stat {
	/*
	 * The pageout code in vmscan.c keeps track of how many of the
224
	 * mem/swap backed and file backed pages are referenced.
225
226
227
228
229
230
231
232
233
	 * The higher the rotated/scanned ratio, the more valuable
	 * that cache is.
	 *
	 * The anon LRU stats live in [0], file LRU stats in [1]
	 */
	unsigned long		recent_rotated[2];
	unsigned long		recent_scanned[2];
};

234
struct lruvec {
235
236
237
238
	struct list_head		lists[NR_LRU_LISTS];
	struct zone_reclaim_stat	reclaim_stat;
	/* Evictions & activations on the inactive file list */
	atomic_long_t			inactive_age;
239
240
	/* Refaults at the time of last reclaim cycle */
	unsigned long			refaults;
Andrew Morton's avatar
Andrew Morton committed
241
#ifdef CONFIG_MEMCG
242
	struct pglist_data *pgdat;
243
#endif
244
245
};

246
247
248
249
250
/* Mask used at gathering information at once (see memcontrol.c) */
#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
#define LRU_ALL	     ((1 << NR_LRU_LISTS) - 1)

251
/* Isolate unmapped file */
252
#define ISOLATE_UNMAPPED	((__force isolate_mode_t)0x2)
253
/* Isolate for asynchronous migration */
254
#define ISOLATE_ASYNC_MIGRATE	((__force isolate_mode_t)0x4)
Minchan Kim's avatar
Minchan Kim committed
255
256
/* Isolate unevictable pages */
#define ISOLATE_UNEVICTABLE	((__force isolate_mode_t)0x8)
257
258

/* LRU Isolation modes. */
259
typedef unsigned __bitwise isolate_mode_t;
260

261
262
263
264
265
266
267
268
269
270
271
enum zone_watermarks {
	WMARK_MIN,
	WMARK_LOW,
	WMARK_HIGH,
	NR_WMARK
};

#define min_wmark_pages(z) (z->watermark[WMARK_MIN])
#define low_wmark_pages(z) (z->watermark[WMARK_LOW])
#define high_wmark_pages(z) (z->watermark[WMARK_HIGH])

Linus Torvalds's avatar
Linus Torvalds committed
272
273
274
275
struct per_cpu_pages {
	int count;		/* number of pages in the list */
	int high;		/* high watermark, emptying needed */
	int batch;		/* chunk size for buddy add/remove */
276
277
278

	/* Lists of pages, one per migrate type stored on the pcp-lists */
	struct list_head lists[MIGRATE_PCPTYPES];
Linus Torvalds's avatar
Linus Torvalds committed
279
280
281
};

struct per_cpu_pageset {
282
	struct per_cpu_pages pcp;
283
284
#ifdef CONFIG_NUMA
	s8 expire;
285
	u16 vm_numa_stat_diff[NR_VM_NUMA_STAT_ITEMS];
286
#endif
287
#ifdef CONFIG_SMP
288
	s8 stat_threshold;
289
290
	s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
#endif
291
};
292

293
294
295
296
297
struct per_cpu_nodestat {
	s8 stat_threshold;
	s8 vm_node_stat_diff[NR_VM_NODE_STAT_ITEMS];
};

298
299
#endif /* !__GENERATING_BOUNDS.H */

300
enum zone_type {
301
#ifdef CONFIG_ZONE_DMA
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
	/*
	 * ZONE_DMA is used when there are devices that are not able
	 * to do DMA to all of addressable memory (ZONE_NORMAL). Then we
	 * carve out the portion of memory that is needed for these devices.
	 * The range is arch specific.
	 *
	 * Some examples
	 *
	 * Architecture		Limit
	 * ---------------------------
	 * parisc, ia64, sparc	<4G
	 * s390			<2G
	 * arm			Various
	 * alpha		Unlimited or 0-16MB.
	 *
	 * i386, x86_64 and multiple other arches
	 * 			<16M.
	 */
	ZONE_DMA,
321
#endif
322
#ifdef CONFIG_ZONE_DMA32
323
324
325
326
327
328
	/*
	 * x86_64 needs two ZONE_DMAs because it supports devices that are
	 * only able to do DMA to the lower 16M but also 32 bit devices that
	 * can only do DMA areas below 4G.
	 */
	ZONE_DMA32,
329
#endif
330
331
332
333
334
335
	/*
	 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
	 * performed on pages in ZONE_NORMAL if the DMA devices support
	 * transfers to all addressable memory.
	 */
	ZONE_NORMAL,
336
#ifdef CONFIG_HIGHMEM
337
338
339
340
341
342
343
344
345
	/*
	 * A memory area that is only addressable by the kernel through
	 * mapping portions into its own address space. This is for example
	 * used by i386 to allow the kernel to address the memory beyond
	 * 900MB. The kernel will set up special mappings (page
	 * table entries on i386) for each page that the kernel needs to
	 * access.
	 */
	ZONE_HIGHMEM,
346
#endif
Mel Gorman's avatar
Mel Gorman committed
347
	ZONE_MOVABLE,
348
349
350
#ifdef CONFIG_ZONE_DEVICE
	ZONE_DEVICE,
#endif
351
	__MAX_NR_ZONES
352

353
};
Linus Torvalds's avatar
Linus Torvalds committed
354

355
356
#ifndef __GENERATING_BOUNDS_H

Linus Torvalds's avatar
Linus Torvalds committed
357
struct zone {
358
	/* Read-mostly fields */
359
360
361
362

	/* zone watermarks, access with *_wmark_pages(zone) macros */
	unsigned long watermark[NR_WMARK];

363
364
	unsigned long nr_reserved_highatomic;

Linus Torvalds's avatar
Linus Torvalds committed
365
	/*
366
367
368
369
370
371
372
	 * We don't know if the memory that we're going to allocate will be
	 * freeable or/and it will be released eventually, so to avoid totally
	 * wasting several GB of ram we must reserve some of the lower zone
	 * memory (otherwise we risk to run OOM on the lower zones despite
	 * there being tons of freeable ram on the higher zones).  This array is
	 * recalculated at runtime if the sysctl_lowmem_reserve_ratio sysctl
	 * changes.
Linus Torvalds's avatar
Linus Torvalds committed
373
	 */
374
	long lowmem_reserve[MAX_NR_ZONES];
375

376
#ifdef CONFIG_NUMA
377
	int node;
378
379
#endif
	struct pglist_data	*zone_pgdat;
380
	struct per_cpu_pageset __percpu *pageset;
381

382
383
#ifndef CONFIG_SPARSEMEM
	/*
384
	 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
385
386
387
388
389
	 * In SPARSEMEM, this map is stored in struct mem_section
	 */
	unsigned long		*pageblock_flags;
#endif /* CONFIG_SPARSEMEM */

Linus Torvalds's avatar
Linus Torvalds committed
390
391
392
	/* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
	unsigned long		zone_start_pfn;

393
	/*
394
395
396
	 * spanned_pages is the total pages spanned by the zone, including
	 * holes, which is calculated as:
	 * 	spanned_pages = zone_end_pfn - zone_start_pfn;
397
	 *
398
399
	 * present_pages is physical pages existing within the zone, which
	 * is calculated as:
400
	 *	present_pages = spanned_pages - absent_pages(pages in holes);
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
	 *
	 * managed_pages is present pages managed by the buddy system, which
	 * is calculated as (reserved_pages includes pages allocated by the
	 * bootmem allocator):
	 *	managed_pages = present_pages - reserved_pages;
	 *
	 * So present_pages may be used by memory hotplug or memory power
	 * management logic to figure out unmanaged pages by checking
	 * (present_pages - managed_pages). And managed_pages should be used
	 * by page allocator and vm scanner to calculate all kinds of watermarks
	 * and thresholds.
	 *
	 * Locking rules:
	 *
	 * zone_start_pfn and spanned_pages are protected by span_seqlock.
	 * It is a seqlock because it has to be read outside of zone->lock,
	 * and it is done in the main allocator path.  But, it is written
	 * quite infrequently.
	 *
	 * The span_seq lock is declared along with zone->lock because it is
421
422
	 * frequently read in proximity to zone->lock.  It's good to
	 * give them a chance of being in the same cacheline.
423
	 *
424
	 * Write access to present_pages at runtime should be protected by
425
426
	 * mem_hotplug_begin/end(). Any reader who can't tolerant drift of
	 * present_pages should get_online_mems() to get a stable value.
427
428
429
430
431
432
	 *
	 * Read access to managed_pages should be safe because it's unsigned
	 * long. Write access to zone->managed_pages and totalram_pages are
	 * protected by managed_page_count_lock at runtime. Idealy only
	 * adjust_managed_page_count() should be used instead of directly
	 * touching zone->managed_pages and totalram_pages.
433
	 */
434
	unsigned long		managed_pages;
435
436
	unsigned long		spanned_pages;
	unsigned long		present_pages;
437
438

	const char		*name;
Linus Torvalds's avatar
Linus Torvalds committed
439

440
441
442
443
444
445
446
447
448
#ifdef CONFIG_MEMORY_ISOLATION
	/*
	 * Number of isolated pageblock. It is used to solve incorrect
	 * freepage counting problem due to racy retrieving migratetype
	 * of pageblock. Protected by zone->lock.
	 */
	unsigned long		nr_isolate_pageblock;
#endif

449
450
451
452
453
#ifdef CONFIG_MEMORY_HOTPLUG
	/* see spanned/present_pages for more description */
	seqlock_t		span_seqlock;
#endif

454
	int initialized;
455

456
	/* Write-intensive fields used from the page allocator */
457
	ZONE_PADDING(_pad1_)
458

459
460
461
462
463
464
	/* free areas of different sizes */
	struct free_area	free_area[MAX_ORDER];

	/* zone flags, see below */
	unsigned long		flags;

465
	/* Primarily protects free_area */
466
467
	spinlock_t		lock;

468
	/* Write-intensive fields used by compaction and vmstats. */
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
	ZONE_PADDING(_pad2_)

	/*
	 * When free pages are below this point, additional steps are taken
	 * when reading the number of free pages to avoid per-cpu counter
	 * drift allowing watermarks to be breached
	 */
	unsigned long percpu_drift_mark;

#if defined CONFIG_COMPACTION || defined CONFIG_CMA
	/* pfn where compaction free scanner should start */
	unsigned long		compact_cached_free_pfn;
	/* pfn where async and sync compaction migration scanner should start */
	unsigned long		compact_cached_migrate_pfn[2];
#endif

#ifdef CONFIG_COMPACTION
	/*
	 * On compaction failure, 1<<compact_defer_shift compactions
	 * are skipped before trying again. The number attempted since
	 * last failure is tracked with compact_considered.
	 */
	unsigned int		compact_considered;
	unsigned int		compact_defer_shift;
	int			compact_order_failed;
#endif

#if defined CONFIG_COMPACTION || defined CONFIG_CMA
	/* Set to true when the PG_migrate_skip bits should be cleared */
	bool			compact_blockskip_flush;
#endif

501
502
	bool			contiguous;

503
504
505
	ZONE_PADDING(_pad3_)
	/* Zone statistics */
	atomic_long_t		vm_stat[NR_VM_ZONE_STAT_ITEMS];
506
	atomic_long_t		vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
507
} ____cacheline_internodealigned_in_smp;
Linus Torvalds's avatar
Linus Torvalds committed
508

509
510
enum pgdat_flags {
	PGDAT_CONGESTED,		/* pgdat has many dirty pages backed by
511
512
					 * a congested BDI
					 */
513
	PGDAT_DIRTY,			/* reclaim scanning has recently found
514
515
516
					 * many dirty file pages at the tail
					 * of the LRU.
					 */
517
	PGDAT_WRITEBACK,		/* reclaim scanning has recently found
518
519
					 * many pages under writeback
					 */
520
	PGDAT_RECLAIM_LOCKED,		/* prevents concurrent reclaim */
Johannes Weiner's avatar
Johannes Weiner committed
521
};
522

Russ Anderson's avatar
Russ Anderson committed
523
static inline unsigned long zone_end_pfn(const struct zone *zone)
524
525
526
527
528
529
530
531
532
{
	return zone->zone_start_pfn + zone->spanned_pages;
}

static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
{
	return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
}

533
534
static inline bool zone_is_initialized(struct zone *zone)
{
535
	return zone->initialized;
536
537
538
539
540
541
542
}

static inline bool zone_is_empty(struct zone *zone)
{
	return zone->spanned_pages == 0;
}

543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
/*
 * Return true if [start_pfn, start_pfn + nr_pages) range has a non-empty
 * intersection with the given zone
 */
static inline bool zone_intersects(struct zone *zone,
		unsigned long start_pfn, unsigned long nr_pages)
{
	if (zone_is_empty(zone))
		return false;
	if (start_pfn >= zone_end_pfn(zone) ||
	    start_pfn + nr_pages <= zone->zone_start_pfn)
		return false;

	return true;
}

Linus Torvalds's avatar
Linus Torvalds committed
559
560
561
562
563
564
565
/*
 * The "priority" of VM scanning is how much of the queues we will scan in one
 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
 * queues ("queue_length >> 12") during an aging round.
 */
#define DEF_PRIORITY 12

566
567
568
/* Maximum number of zones on a zonelist */
#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)

569
570
enum {
	ZONELIST_FALLBACK,	/* zonelist with fallback */
571
#ifdef CONFIG_NUMA
572
573
574
575
576
	/*
	 * The NUMA zonelists are doubled because we need zonelists that
	 * restrict the allocations to a single node for __GFP_THISNODE.
	 */
	ZONELIST_NOFALLBACK,	/* zonelist without fallback (__GFP_THISNODE) */
577
#endif
578
579
	MAX_ZONELISTS
};
580

581
582
583
584
585
586
587
588
589
/*
 * This struct contains information about a zone in a zonelist. It is stored
 * here to avoid dereferences into large structures and lookups of tables
 */
struct zoneref {
	struct zone *zone;	/* Pointer to actual zone */
	int zone_idx;		/* zone_idx(zoneref->zone) */
};

Linus Torvalds's avatar
Linus Torvalds committed
590
591
592
593
594
595
/*
 * One allocation request operates on a zonelist. A zonelist
 * is a list of zones, the first one is the 'goal' of the
 * allocation, the other zones are fallback zones, in decreasing
 * priority.
 *
596
597
598
599
600
601
602
 * To speed the reading of the zonelist, the zonerefs contain the zone index
 * of the entry being read. Helper functions to access information given
 * a struct zoneref are
 *
 * zonelist_zone()	- Return the struct zone * for an entry in _zonerefs
 * zonelist_zone_idx()	- Return the index of the zone for an entry
 * zonelist_node_idx()	- Return the index of the node for an entry
Linus Torvalds's avatar
Linus Torvalds committed
603
604
 */
struct zonelist {
605
	struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
Linus Torvalds's avatar
Linus Torvalds committed
606
607
};

608
609
610
611
612
#ifndef CONFIG_DISCONTIGMEM
/* The array of struct pages - for discontigmem use pgdat->lmem_map */
extern struct page *mem_map;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
613
614
/*
 * On NUMA machines, each NUMA node would have a pg_data_t to describe
615
616
 * it's memory layout. On UMA machines there is a single pglist_data which
 * describes the whole memory.
Linus Torvalds's avatar
Linus Torvalds committed
617
618
619
620
621
622
623
 *
 * Memory statistics and page replacement data structures are maintained on a
 * per-zone basis.
 */
struct bootmem_data;
typedef struct pglist_data {
	struct zone node_zones[MAX_NR_ZONES];
624
	struct zonelist node_zonelists[MAX_ZONELISTS];
Linus Torvalds's avatar
Linus Torvalds committed
625
	int nr_zones;
626
#ifdef CONFIG_FLAT_NODE_MEM_MAP	/* means !SPARSEMEM */
Linus Torvalds's avatar
Linus Torvalds committed
627
	struct page *node_mem_map;
628
629
630
#ifdef CONFIG_PAGE_EXTENSION
	struct page_ext *node_page_ext;
#endif
631
#endif
632
#ifndef CONFIG_NO_BOOTMEM
Linus Torvalds's avatar
Linus Torvalds committed
633
	struct bootmem_data *bdata;
634
#endif
635
636
637
638
639
640
#ifdef CONFIG_MEMORY_HOTPLUG
	/*
	 * Must be held any time you expect node_start_pfn, node_present_pages
	 * or node_spanned_pages stay constant.  Holding this will also
	 * guarantee that any pfn_valid() stays that way.
	 *
641
642
643
	 * pgdat_resize_lock() and pgdat_resize_unlock() are provided to
	 * manipulate node_size_lock without checking for CONFIG_MEMORY_HOTPLUG.
	 *
644
	 * Nests above zone->lock and zone->span_seqlock
645
646
647
	 */
	spinlock_t node_size_lock;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
648
649
650
651
652
653
	unsigned long node_start_pfn;
	unsigned long node_present_pages; /* total number of physical pages */
	unsigned long node_spanned_pages; /* total size of physical page
					     range, including holes */
	int node_id;
	wait_queue_head_t kswapd_wait;
654
	wait_queue_head_t pfmemalloc_wait;
655
656
	struct task_struct *kswapd;	/* Protected by
					   mem_hotplug_begin/end() */
657
658
659
	int kswapd_order;
	enum zone_type kswapd_classzone_idx;

660
661
	int kswapd_failures;		/* Number of 'reclaimed == 0' runs */

662
663
664
665
666
667
#ifdef CONFIG_COMPACTION
	int kcompactd_max_order;
	enum zone_type kcompactd_classzone_idx;
	wait_queue_head_t kcompactd_wait;
	struct task_struct *kcompactd;
#endif
668
#ifdef CONFIG_NUMA_BALANCING
669
	/* Lock serializing the migrate rate limiting window */
670
671
672
673
674
675
676
677
	spinlock_t numabalancing_migrate_lock;

	/* Rate limiting time interval */
	unsigned long numabalancing_migrate_next_window;

	/* Number of pages migrated during the rate limiting time interval */
	unsigned long numabalancing_migrate_nr_pages;
#endif
678
679
680
681
682
683
	/*
	 * This is a per-node reserve of pages that are not available
	 * to userspace allocations.
	 */
	unsigned long		totalreserve_pages;

684
685
686
687
688
689
690
691
#ifdef CONFIG_NUMA
	/*
	 * zone reclaim becomes active if more unmapped pages exist.
	 */
	unsigned long		min_unmapped_pages;
	unsigned long		min_slab_pages;
#endif /* CONFIG_NUMA */

692
693
694
	/* Write-intensive fields used by page reclaim */
	ZONE_PADDING(_pad1_)
	spinlock_t		lru_lock;
695
696
697
698
699
700
701

#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
	/*
	 * If memory initialisation on large machines is deferred then this
	 * is the first PFN that needs to be initialised.
	 */
	unsigned long first_deferred_pfn;
702
	unsigned long static_init_size;
703
#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
704
705
706
707
708
709

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	spinlock_t split_queue_lock;
	struct list_head split_queue;
	unsigned long split_queue_len;
#endif
710

711
712
713
714
715
716
717
718
719
720
721
722
723
	/* Fields commonly accessed by the page reclaim scanner */
	struct lruvec		lruvec;

	/*
	 * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on
	 * this node's LRU.  Maintained by the pageout code.
	 */
	unsigned int inactive_ratio;

	unsigned long		flags;

	ZONE_PADDING(_pad2_)

724
725
726
	/* Per-node vmstats */
	struct per_cpu_nodestat __percpu *per_cpu_nodestats;
	atomic_long_t		vm_stat[NR_VM_NODE_STAT_ITEMS];
Linus Torvalds's avatar
Linus Torvalds committed
727
728
729
730
} pg_data_t;

#define node_present_pages(nid)	(NODE_DATA(nid)->node_present_pages)
#define node_spanned_pages(nid)	(NODE_DATA(nid)->node_spanned_pages)
731
#ifdef CONFIG_FLAT_NODE_MEM_MAP
732
#define pgdat_page_nr(pgdat, pagenr)	((pgdat)->node_mem_map + (pagenr))
733
734
735
#else
#define pgdat_page_nr(pgdat, pagenr)	pfn_to_page((pgdat)->node_start_pfn + (pagenr))
#endif
736
#define nid_page_nr(nid, pagenr) 	pgdat_page_nr(NODE_DATA(nid),(pagenr))
Linus Torvalds's avatar
Linus Torvalds committed
737

738
#define node_start_pfn(nid)	(NODE_DATA(nid)->node_start_pfn)
739
#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
740
741
742
743
static inline spinlock_t *zone_lru_lock(struct zone *zone)
{
	return &zone->zone_pgdat->lru_lock;
}
744

745
static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
746
{
747
	return &pgdat->lruvec;
748
749
}

750
751
752
753
754
755
756
757
758
static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
{
	return pgdat->node_start_pfn + pgdat->node_spanned_pages;
}

static inline bool pgdat_is_empty(pg_data_t *pgdat)
{
	return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
}
759

760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
static inline int zone_id(const struct zone *zone)
{
	struct pglist_data *pgdat = zone->zone_pgdat;

	return zone - pgdat->node_zones;
}

#ifdef CONFIG_ZONE_DEVICE
static inline bool is_dev_zone(const struct zone *zone)
{
	return zone_id(zone) == ZONE_DEVICE;
}
#else
static inline bool is_dev_zone(const struct zone *zone)
{
	return false;
}
#endif

779
780
#include <linux/memory_hotplug.h>

781
void build_all_zonelists(pg_data_t *pgdat);
782
void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx);
783
784
785
bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
			 int classzone_idx, unsigned int alloc_flags,
			 long free_pages);
786
bool zone_watermark_ok(struct zone *z, unsigned int order,
787
788
		unsigned long mark, int classzone_idx,
		unsigned int alloc_flags);
789
bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
790
		unsigned long mark, int classzone_idx);
Dave Hansen's avatar
Dave Hansen committed
791
792
793
794
enum memmap_context {
	MEMMAP_EARLY,
	MEMMAP_HOTPLUG,
};
795
extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
796
				     unsigned long size);
797

798
extern void lruvec_init(struct lruvec *lruvec);
799

800
static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
801
{
Andrew Morton's avatar
Andrew Morton committed
802
#ifdef CONFIG_MEMCG
803
	return lruvec->pgdat;
804
#else
805
	return container_of(lruvec, struct pglist_data, lruvec);
806
807
808
#endif
}

809
extern unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
810

Linus Torvalds's avatar
Linus Torvalds committed
811
812
813
814
815
816
#ifdef CONFIG_HAVE_MEMORY_PRESENT
void memory_present(int nid, unsigned long start, unsigned long end);
#else
static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
#endif

817
818
819
820
821
822
#ifdef CONFIG_HAVE_MEMORYLESS_NODES
int local_memory_node(int node_id);
#else
static inline int local_memory_node(int node_id) { return node_id; };
#endif

Linus Torvalds's avatar
Linus Torvalds committed
823
824
825
826
827
828
829
830
831
#ifdef CONFIG_NEED_NODE_MEMMAP_SIZE
unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
#endif

/*
 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
 */
#define zone_idx(zone)		((zone) - (zone)->zone_pgdat->node_zones)

832
833
834
835
836
837
838
839
840
841
842
843
844
/*
 * Returns true if a zone has pages managed by the buddy allocator.
 * All the reclaim decisions have to use this function rather than
 * populated_zone(). If the whole zone is reserved then we can easily
 * end up with populated_zone() && !managed_zone().
 */
static inline bool managed_zone(struct zone *zone)
{
	return zone->managed_pages;
}

/* Returns true if a zone has memory */
static inline bool populated_zone(struct zone *zone)
845
{
846
	return zone->present_pages;
847
848
}

Mel Gorman's avatar
Mel Gorman committed
849
850
extern int movable_zone;

851
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
852
853
static inline int zone_movable_is_highmem(void)
{
854
#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gorman's avatar
Mel Gorman committed
855
856
	return movable_zone == ZONE_HIGHMEM;
#else
857
	return (ZONE_MOVABLE - 1) == ZONE_HIGHMEM;
Mel Gorman's avatar
Mel Gorman committed
858
859
#endif
}
860
#endif
Mel Gorman's avatar
Mel Gorman committed
861

862
static inline int is_highmem_idx(enum zone_type idx)
Linus Torvalds's avatar
Linus Torvalds committed
863
{
864
#ifdef CONFIG_HIGHMEM
Mel Gorman's avatar
Mel Gorman committed
865
866
	return (idx == ZONE_HIGHMEM ||
		(idx == ZONE_MOVABLE && zone_movable_is_highmem()));
867
868
869
#else
	return 0;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
870
871
872
873
874
875
876
877
878
879
}

/**
 * is_highmem - helper function to quickly check if a struct zone is a 
 *              highmem zone or not.  This is an attempt to keep references
 *              to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
 * @zone - pointer to struct zone variable
 */
static inline int is_highmem(struct zone *zone)
{
880
#ifdef CONFIG_HIGHMEM
Chanho Min's avatar
Chanho Min committed
881
	return is_highmem_idx(zone_idx(zone));
882
883
884
#else
	return 0;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
885
886
887
888
}

/* These two functions are used to setup the per zone pages min values */
struct ctl_table;
889
int min_free_kbytes_sysctl_handler(struct ctl_table *, int,
Linus Torvalds's avatar
Linus Torvalds committed
890
					void __user *, size_t *, loff_t *);
891
892
int watermark_scale_factor_sysctl_handler(struct ctl_table *, int,
					void __user *, size_t *, loff_t *);
Linus Torvalds's avatar
Linus Torvalds committed
893
extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
894
int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int,
Linus Torvalds's avatar
Linus Torvalds committed
895
					void __user *, size_t *, loff_t *);
896
int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int,
897
					void __user *, size_t *, loff_t *);
898
int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
899
			void __user *, size_t *, loff_t *);
900
int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
901
			void __user *, size_t *, loff_t *);
Linus Torvalds's avatar
Linus Torvalds committed
902

903
extern int numa_zonelist_order_handler(struct ctl_table *, int,
904
			void __user *, size_t *, loff_t *);
905
extern char numa_zonelist_order[];
906
#define NUMA_ZONELIST_ORDER_LEN	16
907

908
#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds's avatar
Linus Torvalds committed
909
910
911
912
913

extern struct pglist_data contig_page_data;
#define NODE_DATA(nid)		(&contig_page_data)
#define NODE_MEM_MAP(nid)	mem_map

914
#else /* CONFIG_NEED_MULTIPLE_NODES */
Linus Torvalds's avatar
Linus Torvalds committed
915
916
917

#include <asm/mmzone.h>

918
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
919

920
921
922
extern struct pglist_data *first_online_pgdat(void);
extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
extern struct zone *next_zone(struct zone *zone);
923
924

/**
925
 * for_each_online_pgdat - helper macro to iterate over all online nodes
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
 * @pgdat - pointer to a pg_data_t variable
 */
#define for_each_online_pgdat(pgdat)			\
	for (pgdat = first_online_pgdat();		\
	     pgdat;					\
	     pgdat = next_online_pgdat(pgdat))
/**
 * for_each_zone - helper macro to iterate over all memory zones
 * @zone - pointer to struct zone variable
 *
 * The user only needs to declare the zone variable, for_each_zone
 * fills it in.
 */
#define for_each_zone(zone)			        \
	for (zone = (first_online_pgdat())->node_zones; \
	     zone;					\
	     zone = next_zone(zone))

944
945
946
947
948
949
950
951
#define for_each_populated_zone(zone)		        \
	for (zone = (first_online_pgdat())->node_zones; \
	     zone;					\
	     zone = next_zone(zone))			\
		if (!populated_zone(zone))		\
			; /* do nothing */		\
		else

952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
static inline struct zone *zonelist_zone(struct zoneref *zoneref)
{
	return zoneref->zone;
}

static inline int zonelist_zone_idx(struct zoneref *zoneref)
{
	return zoneref->zone_idx;
}

static inline int zonelist_node_idx(struct zoneref *zoneref)
{
#ifdef CONFIG_NUMA
	/* zone_to_nid not available in this context */
	return zoneref->zone->node;
#else
	return 0;
#endif /* CONFIG_NUMA */
}

972
973
974
975
struct zoneref *__next_zones_zonelist(struct zoneref *z,
					enum zone_type highest_zoneidx,
					nodemask_t *nodes);

976
977
978
979
980
981
982
983
/**
 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
 * @z - The cursor used as a starting point for the search
 * @highest_zoneidx - The zone index of the highest zone to return
 * @nodes - An optional nodemask to filter the zonelist with
 *
 * This function returns the next zone at or below a given zone index that is
 * within the allowed nodemask using a cursor as the starting point for the
984
985
986
 * search. The zoneref returned is a cursor that represents the current zone
 * being examined. It should be advanced by one before calling
 * next_zones_zonelist again.
987
 */
988
static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z,
989
					enum zone_type highest_zoneidx,
990
991
992
993
994
995
					nodemask_t *nodes)
{
	if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx))
		return z;
	return __next_zones_zonelist(z, highest_zoneidx, nodes);
}
996

997
998
999
1000
1001
/**
 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
 * @zonelist - The zonelist to search for a suitable zone
 * @highest_zoneidx - The zone index of the highest zone to return
 * @nodes - An optional nodemask to filter the zonelist with
1002
 * @return - Zoneref pointer for the first suitable zone found (see below)
1003
1004
1005
 *
 * This function returns the first zone at or below a given zone index that is
 * within the allowed nodemask. The zoneref returned is a cursor that can be
1006
1007
 * used to iterate the zonelist with next_zones_zonelist by advancing it by
 * one before calling.
1008
1009
1010
1011
 *
 * When no eligible zone is found, zoneref->zone is NULL (zoneref itself is
 * never NULL). This may happen either genuinely, or due to concurrent nodemask
 * update due to cpuset modification.
1012
 */
1013
static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
1014
					enum zone_type highest_zoneidx,
1015
					nodemask_t *nodes)
1016
{
1017
	return next_zones_zonelist(zonelist->_zonerefs,
1018
							highest_zoneidx, nodes);
1019
1020
}

1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
/**
 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
 * @zone - The current zone in the iterator
 * @z - The current pointer within zonelist->zones being iterated
 * @zlist - The zonelist being iterated
 * @highidx - The zone index of the highest zone to return
 * @nodemask - Nodemask allowed by the allocator
 *
 * This iterator iterates though all zones at or below a given zone index and
 * within a given nodemask
 */
#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
1033
	for (z = first_zones_zonelist(zlist, highidx, nodemask), zone = zonelist_zone(z);	\
1034
		zone;							\
1035
		z = next_zones_zonelist(++z, highidx, nodemask),	\
1036
1037
1038
1039
1040
1041
1042
1043
			zone = zonelist_zone(z))

#define for_next_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
	for (zone = z->zone;	\
		zone;							\
		z = next_zones_zonelist(++z, highidx, nodemask),	\
			zone = zonelist_zone(z))

1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054

/**
 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
 * @zone - The current zone in the iterator
 * @z - The current pointer within zonelist->zones being iterated
 * @zlist - The zonelist being iterated
 * @highidx - The zone index of the highest zone to return
 *
 * This iterator iterates though all zones at or below a given zone index.
 */
#define for_each_zone_zonelist(zone, z, zlist, highidx) \
1055
	for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
1056

1057
1058
1059
1060
#ifdef CONFIG_SPARSEMEM
#include <asm/sparsemem.h>
#endif

1061
#if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
Tejun Heo's avatar
Tejun Heo committed
1062
	!defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP)
1063
1064
static inline unsigned long early_pfn_to_nid(unsigned long pfn)
{
1065
	BUILD_BUG_ON(IS_ENABLED(CONFIG_NUMA));
1066
1067
	return 0;
}
1068
1069
#endif

1070
1071
1072
1073
#ifdef CONFIG_FLATMEM
#define pfn_to_nid(pfn)		(0)
#endif

1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
#ifdef CONFIG_SPARSEMEM

/*
 * SECTION_SHIFT    		#bits space required to store a section #
 *
 * PA_SECTION_SHIFT		physical address to/from section number
 * PFN_SECTION_SHIFT		pfn to/from section number
 */
#define PA_SECTION_SHIFT	(SECTION_SIZE_BITS)
#define PFN_SECTION_SHIFT	(SECTION_SIZE_BITS - PAGE_SHIFT)

#define NR_MEM_SECTIONS		(1UL << SECTIONS_SHIFT)

#define PAGES_PER_SECTION       (1UL << PFN_SECTION_SHIFT)
#define PAGE_SECTION_MASK	(~(PAGES_PER_SECTION-1))

1090
#define SECTION_BLOCKFLAGS_BITS \
1091
	((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
1092

1093
1094
1095
1096
#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
#error Allocator MAX_ORDER exceeds SECTION_SIZE
#endif

1097
1098
1099
#define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
#define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)

1100
1101
1102
#define SECTION_ALIGN_UP(pfn)	(((pfn) + PAGES_PER_SECTION - 1) & PAGE_SECTION_MASK)
#define SECTION_ALIGN_DOWN(pfn)	((pfn) & PAGE_SECTION_MASK)

1103
struct page;
1104
struct page_ext;
1105
struct mem_section {
1106
1107
1108
1109
1110
	/*
	 * This is, logically, a pointer to an array of struct
	 * pages.  However, it is stored with some other magic.
	 * (see sparse.c::sparse_init_one_section())
	 *
1111
1112
1113
1114
	 * Additionally during early boot we encode node id of
	 * the location of the section here to guide allocation.
	 * (see sparse.c::memory_present())
	 *
1115
1116
1117
1118
	 * Making it a UL at least makes someone do a cast
	 * before using it wrong.
	 */
	unsigned long section_mem_map;
1119
1120
1121

	/* See declaration of similar field in struct zone */
	unsigned long *pageblock_flags;
1122
1123
#ifdef CONFIG_PAGE_EXTENSION
	/*
1124
	 * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use
1125
1126
1127
1128
1129
	 * section. (see page_ext.h about this.)
	 */
	struct page_ext *page_ext;
	unsigned long pad;
#endif
1130
1131
1132
1133
	/*
	 * WARNING: mem_section must be a power-of-2 in size for the
	 * calculation and use of SECTION_ROOT_MASK to make sense.
	 */
1134
1135
};

1136
1137
1138
1139
1140
#ifdef CONFIG_SPARSEMEM_EXTREME
#define SECTIONS_PER_ROOT       (PAGE_SIZE / sizeof (struct mem_section))
#else
#define SECTIONS_PER_ROOT	1
#endif
Bob Picco's avatar
Bob Picco committed
1141

1142
#define SECTION_NR_TO_ROOT(sec)	((sec) / SECTIONS_PER_ROOT)
1143
#define NR_SECTION_ROOTS	DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT)
1144
#define SECTION_ROOT_MASK	(SECTIONS_PER_ROOT - 1)
Bob Picco's avatar
Bob Picco committed
1145

1146
1147
#ifdef CONFIG_SPARSEMEM_EXTREME
extern struct mem_section *mem_section[NR_SECTION_ROOTS];
Bob Picco's avatar
Bob Picco committed
1148
#else
1149
1150
extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
#endif
1151

1152
1153
static inline struct mem_section *__nr_to_section(unsigned long nr)
{
1154
1155
1156
	if (!mem_section[SECTION_NR_TO_ROOT(nr)])
		return NULL;
	return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
1157
}
1158
extern int __section_nr(struct mem_section* ms);
1159
extern unsigned long usemap_size(void);
1160
1161
1162
1163
1164
1165
1166
1167

/*
 * We use the lower bits of the mem_map pointer to store
 * a little bit of information.  There should be at least
 * 3 bits here due to 32-bit alignment.
 */
#define	SECTION_MARKED_PRESENT	(1UL<<0)
#define SECTION_HAS_MEM_MAP	(1UL<<1)
1168
1169
#define SECTION_IS_ONLINE	(1UL<<2)
#define SECTION_MAP_LAST_BIT	(1UL<<3)
1170
#define SECTION_MAP_MASK	(~(SECTION_MAP_LAST_BIT-1))
1171
#define SECTION_NID_SHIFT	3
1172
1173
1174
1175
1176
1177
1178
1179

static inline struct page *__section_mem_map_addr(struct mem_section *section)
{
	unsigned long map = section->section_mem_map;
	map &= SECTION_MAP_MASK;
	return (struct page *)map;
}

1180
static inline int present_section(struct mem_section *section)
1181
{
Bob Picco's avatar
Bob Picco committed
1182
	return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
1183
1184
}

1185
1186
1187
1188
1189
1190
static inline int present_section_nr(unsigned long nr)
{
	return present_section(__nr_to_section(nr));
}

static inline int valid_section(struct mem_section *section)
1191
{
Bob Picco's avatar
Bob Picco committed
1192
	return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
1193
1194
1195
1196
1197
1198
1199
}

static inline int valid_section_nr(unsigned long nr)
{
	return valid_section(__nr_to_section(nr));
}

1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
static inline int online_section(struct mem_section *section)
{
	return (section && (section->section_mem_map & SECTION_IS_ONLINE));
}

static inline int online_section_nr(unsigned long nr)
{
	return online_section(__nr_to_section(nr));
}

#ifdef CONFIG_MEMORY_HOTPLUG
void online_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
#ifdef CONFIG_MEMORY_HOTREMOVE
void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn);
#endif
#endif

1217
1218
static inline struct mem_section *__pfn_to_section(unsigned long pfn)
{
1219
	return __nr_to_section(pfn_to_section_nr(pfn));
1220
1221
}

1222
1223
extern int __highest_present_section_nr;

1224
#ifndef CONFIG_HAVE_ARCH_PFN_VALID
1225
1226
1227
1228
static inline int pfn_valid(unsigned long pfn)
{
	if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
		return 0;
1229
	return valid_section(__nr_to_section(pfn_to_section_nr(pfn)));
1230
}
1231
#endif
1232