bootmem.c 14.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
39
/* return the number of _pages_ that will be allocated for the boot bitmap */
40
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
47
48
49
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
50

51
52
53
/*
 * link bdata in order
 */
54
static void __init link_bootmem(bootmem_data_t *bdata)
55
56
{
	bootmem_data_t *ent;
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

72
73
74
75
76
77
78
79
80
81
82
83
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
Linus Torvalds's avatar
Linus Torvalds committed
84
85
86
87

/*
 * Called once to set up the allocator itself.
 */
88
static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
Linus Torvalds's avatar
Linus Torvalds committed
89
90
91
	unsigned long mapstart, unsigned long start, unsigned long end)
{
	bootmem_data_t *bdata = pgdat->bdata;
92
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
93

94
95
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
96
	bdata->node_low_pfn = end;
97
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
98
99
100
101
102

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
103
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
104
105
106
107
108
109
110
111
112
113
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
114
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
115
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
116
{
117
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
118
	unsigned long i;
119
120
121
122
123
124
125

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
126

Linus Torvalds's avatar
Linus Torvalds committed
127
	/*
128
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
129
	 */
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
156
	BUG_ON(!size);
157

158
159
160
161
162
163
164
165
166
167
168
169
170
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

171
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
172
173
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
174

175
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
176
177
178
179
180
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
181
	}
Linus Torvalds's avatar
Linus Torvalds committed
182
183
}

184
185
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
186
{
187
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
188
	unsigned long i;
189

190
191
192
193
194
195
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
196
197
198
199
200
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

201
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
Linus Torvalds's avatar
Linus Torvalds committed
202
203
204
		bdata->last_success = addr;

	/*
205
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
206
	 */
207
208
209
210
211
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

212
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
213
214
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
235
void * __init
Linus Torvalds's avatar
Linus Torvalds committed
236
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
237
	      unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
238
{
239
	unsigned long areasize, preferred;
240
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
241
	void *ret;
242
243
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
244

245
	if (!size) {
Linus Torvalds's avatar
Linus Torvalds committed
246
247
248
249
250
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

251
252
253
254
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

255
256
257
258
259
260
261
262
263
264
265
266
267
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

268
269
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
270
271
272
	if (limit && end_pfn > limit)
		end_pfn = limit;

273
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
279
280
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
281
282
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
283

284
285
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
286
			if (!limit || (limit && limit > bdata->last_success))
287
				preferred = bdata->last_success - node_boot_start;
288
	}
Linus Torvalds's avatar
Linus Torvalds committed
289

290
	preferred = PFN_DOWN(ALIGN(preferred, align));
291
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
295
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
296
		unsigned long j;
297

298
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
299
		i = ALIGN(i, incr);
300
301
		if (i >= eidx)
			break;
302
		if (test_bit(i, node_bootmem_map)) {
303
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
304
			continue;
305
		}
Linus Torvalds's avatar
Linus Torvalds committed
306
307
308
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
309
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
310
311
312
313
314
315
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
316
317
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
318
319
	}

320
321
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
322
323
324
325
326
		goto restart_scan;
	}
	return NULL;

found:
327
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
328
329
330
331
332
333
334
335
336
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
337
		unsigned long offset, remaining_size;
338
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
339
		BUG_ON(offset > PAGE_SIZE);
340
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
341
342
343
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
344
345
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
346
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
347
348
		} else {
			remaining_size = size - remaining_size;
349
350
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
351
					   offset + node_boot_start);
352
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
353
354
355
356
357
358
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
359
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
360
361
362
363
364
	}

	/*
	 * Reserve the area now:
	 */
365
	for (i = start; i < start + areasize; i++)
366
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
367
368
369
370
371
372
373
374
			BUG();
	memset(ret, 0, size);
	return ret;
}

static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
{
	struct page *page;
375
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
376
377
378
379
380
381
382
383
384
385
	bootmem_data_t *bdata = pgdat->bdata;
	unsigned long i, count, total = 0;
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
386
387
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
Linus Torvalds's avatar
Linus Torvalds committed
388
389
390
391
392
393
394
	map = bdata->node_bootmem_map;
	/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
	if (bdata->node_boot_start == 0 ||
	    ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
		gofast = 1;
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
395

Linus Torvalds's avatar
Linus Torvalds committed
396
		if (gofast && v == ~0UL) {
397
			int order;
Linus Torvalds's avatar
Linus Torvalds committed
398

399
			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
400
401
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
402
			__free_pages_bootmem(page, order);
Linus Torvalds's avatar
Linus Torvalds committed
403
404
405
406
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
407
408

			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
409
410
411
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
412
					__free_pages_bootmem(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
413
414
415
				}
			}
		} else {
416
			i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
417
		}
418
		pfn += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
419
420
421
422
423
424
425
426
427
	}
	total += count;

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
	count = 0;
428
429
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
	for (i = 0; i < idx; i++, page++) {
430
		__free_pages_bootmem(page, 0);
431
		count++;
Linus Torvalds's avatar
Linus Torvalds committed
432
433
434
435
436
437
438
	}
	total += count;
	bdata->node_bootmem_map = NULL;

	return total;
}

439
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
440
				unsigned long startpfn, unsigned long endpfn)
Linus Torvalds's avatar
Linus Torvalds committed
441
{
442
	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
Linus Torvalds's avatar
Linus Torvalds committed
443
444
}

445
void __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
446
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
447
{
448
449
450
451
452
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
		return;
453
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
Linus Torvalds's avatar
Linus Torvalds committed
454
455
}

456
457
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
458
459
460
461
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

462
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
Linus Torvalds's avatar
Linus Torvalds committed
463
{
464
	register_page_bootmem_info_node(pgdat);
465
	return free_all_bootmem_core(pgdat);
Linus Torvalds's avatar
Linus Torvalds committed
466
467
}

468
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
469
470
471
{
	max_low_pfn = pages;
	min_low_pfn = start;
472
	return init_bootmem_core(NODE_DATA(0), start, 0, pages);
Linus Torvalds's avatar
Linus Torvalds committed
473
474
475
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
476
477
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
Linus Torvalds's avatar
Linus Torvalds committed
478
{
479
480
481
482
483
484
485
486
487
488
489
490
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
491
492
493
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

494
void __init free_bootmem(unsigned long addr, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
495
{
496
497
498
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
Linus Torvalds's avatar
Linus Torvalds committed
499
500
}

501
unsigned long __init free_all_bootmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
502
{
503
	return free_all_bootmem_core(NODE_DATA(0));
Linus Torvalds's avatar
Linus Torvalds committed
504
505
}

506
507
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
508
{
509
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
510
511
	void *ptr;

512
513
514
515
516
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
517
518
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
519

520
521
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
522
523
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
524

525
526
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
527
528
529
530
531
532
533
534
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

535

536
537
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
538
539
540
{
	void *ptr;

541
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
542
	if (ptr)
543
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
544

545
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
546
547
}

548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

579
580
581
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
582

583
584
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
585
{
586
	bootmem_data_t *bdata;
587
588
	void *ptr;

589
	list_for_each_entry(bdata, &bdata_list, list) {
590
591
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
592
593
594
		if (ptr)
			return ptr;
	}
595
596
597
598
599
600
601
602
603
604
605
606

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
607
608
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
609
}