bootmem.c 15 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
39
/* return the number of _pages_ that will be allocated for the boot bitmap */
40
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
47
48
49
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
50

51
52
53
/*
 * link bdata in order
 */
54
static void __init link_bootmem(bootmem_data_t *bdata)
55
56
{
	bootmem_data_t *ent;
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

72
73
74
75
76
77
78
79
80
81
82
83
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
Linus Torvalds's avatar
Linus Torvalds committed
84
85
86
87

/*
 * Called once to set up the allocator itself.
 */
88
static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
Linus Torvalds's avatar
Linus Torvalds committed
89
90
91
	unsigned long mapstart, unsigned long start, unsigned long end)
{
	bootmem_data_t *bdata = pgdat->bdata;
92
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
93

94
	mminit_validate_memmodel_limits(&start, &end);
95
96
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
97
	bdata->node_low_pfn = end;
98
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
99
100
101
102
103

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
104
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
105
106
107
108
109
110
111
112
113
114
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
115
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
116
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
117
{
118
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
119
	unsigned long i;
120
121
122
123
124
125
126

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
127

Linus Torvalds's avatar
Linus Torvalds committed
128
	/*
129
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
130
	 */
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
157
	BUG_ON(!size);
158

159
160
161
162
163
164
165
166
167
168
169
170
171
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

172
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
173
174
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
175

176
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
177
178
179
180
181
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
182
	}
Linus Torvalds's avatar
Linus Torvalds committed
183
184
}

185
186
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
187
{
188
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
189
	unsigned long i;
190

191
192
193
194
195
196
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
197
198
199
200
201
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

202
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
Linus Torvalds's avatar
Linus Torvalds committed
203
204
205
		bdata->last_success = addr;

	/*
206
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
207
	 */
208
209
210
211
212
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

213
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
214
215
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
236
void * __init
Linus Torvalds's avatar
Linus Torvalds committed
237
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
238
	      unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
239
{
240
	unsigned long areasize, preferred;
241
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
242
	void *ret;
243
244
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
245

246
	if (!size) {
Linus Torvalds's avatar
Linus Torvalds committed
247
248
249
250
251
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

252
253
254
255
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

256
257
258
259
260
261
262
263
264
265
266
267
268
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

269
270
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
271
272
273
	if (limit && end_pfn > limit)
		end_pfn = limit;

274
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
275
276
277
278
279

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
280
281
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
282
283
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
284

285
286
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
287
			if (!limit || (limit && limit > bdata->last_success))
288
				preferred = bdata->last_success - node_boot_start;
289
	}
Linus Torvalds's avatar
Linus Torvalds committed
290

291
	preferred = PFN_DOWN(ALIGN(preferred, align));
292
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
293
294
295
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
296
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
297
		unsigned long j;
298

299
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
300
		i = ALIGN(i, incr);
301
302
		if (i >= eidx)
			break;
303
		if (test_bit(i, node_bootmem_map)) {
304
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
305
			continue;
306
		}
Linus Torvalds's avatar
Linus Torvalds committed
307
308
309
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
310
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
311
312
313
314
315
316
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
317
318
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
319
320
	}

321
322
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
323
324
325
326
327
		goto restart_scan;
	}
	return NULL;

found:
328
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
329
330
331
332
333
334
335
336
337
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
338
		unsigned long offset, remaining_size;
339
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
340
		BUG_ON(offset > PAGE_SIZE);
341
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
342
343
344
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
345
346
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
347
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
348
349
		} else {
			remaining_size = size - remaining_size;
350
351
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
352
					   offset + node_boot_start);
353
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
354
355
356
357
358
359
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
360
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
361
362
363
364
365
	}

	/*
	 * Reserve the area now:
	 */
366
	for (i = start; i < start + areasize; i++)
367
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
368
369
370
371
372
373
374
375
			BUG();
	memset(ret, 0, size);
	return ret;
}

static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
{
	struct page *page;
376
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
377
378
379
380
381
382
383
384
385
386
	bootmem_data_t *bdata = pgdat->bdata;
	unsigned long i, count, total = 0;
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
387
388
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
Linus Torvalds's avatar
Linus Torvalds committed
389
390
391
392
393
394
395
	map = bdata->node_bootmem_map;
	/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
	if (bdata->node_boot_start == 0 ||
	    ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
		gofast = 1;
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
396

Linus Torvalds's avatar
Linus Torvalds committed
397
		if (gofast && v == ~0UL) {
398
			int order;
Linus Torvalds's avatar
Linus Torvalds committed
399

400
			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
401
402
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
403
			__free_pages_bootmem(page, order);
Linus Torvalds's avatar
Linus Torvalds committed
404
405
406
407
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
408
409

			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
410
411
412
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
413
					__free_pages_bootmem(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
414
415
416
				}
			}
		} else {
417
			i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
418
		}
419
		pfn += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
420
421
422
423
424
425
426
427
428
	}
	total += count;

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
	count = 0;
429
430
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
	for (i = 0; i < idx; i++, page++) {
431
		__free_pages_bootmem(page, 0);
432
		count++;
Linus Torvalds's avatar
Linus Torvalds committed
433
434
435
436
437
438
439
	}
	total += count;
	bdata->node_bootmem_map = NULL;

	return total;
}

440
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
441
				unsigned long startpfn, unsigned long endpfn)
Linus Torvalds's avatar
Linus Torvalds committed
442
{
443
	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
Linus Torvalds's avatar
Linus Torvalds committed
444
445
}

446
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
447
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
448
{
449
450
451
452
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
453
		return -ENOMEM;
454
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
455
456

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
457
458
}

459
460
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
461
462
463
464
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

465
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
Linus Torvalds's avatar
Linus Torvalds committed
466
{
467
	register_page_bootmem_info_node(pgdat);
468
	return free_all_bootmem_core(pgdat);
Linus Torvalds's avatar
Linus Torvalds committed
469
470
}

471
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
472
473
474
{
	max_low_pfn = pages;
	min_low_pfn = start;
475
	return init_bootmem_core(NODE_DATA(0), start, 0, pages);
Linus Torvalds's avatar
Linus Torvalds committed
476
477
478
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
479
480
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
Linus Torvalds's avatar
Linus Torvalds committed
481
{
482
483
484
485
486
487
488
489
490
491
492
493
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
494
495
496
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

497
void __init free_bootmem(unsigned long addr, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
498
{
499
500
501
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
Linus Torvalds's avatar
Linus Torvalds committed
502
503
}

504
unsigned long __init free_all_bootmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
505
{
506
	return free_all_bootmem_core(NODE_DATA(0));
Linus Torvalds's avatar
Linus Torvalds committed
507
508
}

509
510
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
511
{
512
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
513
514
	void *ptr;

515
516
517
518
519
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
520
521
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
522

523
524
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
525
526
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
527

528
529
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
530
531
532
533
534
535
536
537
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

538

539
540
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
541
542
543
{
	void *ptr;

544
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
545
	if (ptr)
546
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
547

548
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
549
550
}

551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

582
583
584
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
585

586
587
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
588
{
589
	bootmem_data_t *bdata;
590
591
	void *ptr;

592
	list_for_each_entry(bdata, &bdata_list, list) {
593
594
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
595
596
597
		if (ptr)
			return ptr;
	}
598
599
600
601
602
603
604
605
606
607
608
609

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
610
611
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
612
}