bootmem.c 14.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
39
/* return the number of _pages_ that will be allocated for the boot bitmap */
40
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
41
42
43
44
45
46
47
48
49
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
50

51
52
53
/*
 * link bdata in order
 */
54
static void __init link_bootmem(bootmem_data_t *bdata)
55
56
{
	bootmem_data_t *ent;
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

72
73
74
75
76
77
78
79
80
81
82
83
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
Linus Torvalds's avatar
Linus Torvalds committed
84
85
86
87

/*
 * Called once to set up the allocator itself.
 */
88
static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
Linus Torvalds's avatar
Linus Torvalds committed
89
90
91
	unsigned long mapstart, unsigned long start, unsigned long end)
{
	bootmem_data_t *bdata = pgdat->bdata;
92
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
93

94
95
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
96
	bdata->node_low_pfn = end;
97
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
98
99
100
101
102

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
103
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
104
105
106
107
108
109
110
111
112
113
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
114
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
115
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
116
{
117
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
118
	unsigned long i;
119
120
121
122
123
124
125

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
126

Linus Torvalds's avatar
Linus Torvalds committed
127
	/*
128
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
129
	 */
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
156
	BUG_ON(!size);
157

158
159
160
161
162
163
164
165
166
167
168
169
170
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

171
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
172
173
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
174

175
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
176
177
178
179
180
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
181
	}
Linus Torvalds's avatar
Linus Torvalds committed
182
183
}

184
185
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
186
{
187
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
188
	unsigned long i;
189

190
191
192
193
194
195
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
196
197
198
199
200
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

201
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
Linus Torvalds's avatar
Linus Torvalds committed
202
203
204
		bdata->last_success = addr;

	/*
205
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
206
	 */
207
208
209
210
211
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

212
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
213
214
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
235
void * __init
Linus Torvalds's avatar
Linus Torvalds committed
236
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
237
	      unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
238
{
239
	unsigned long areasize, preferred;
240
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
241
	void *ret;
242
243
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
244

245
	if (!size) {
Linus Torvalds's avatar
Linus Torvalds committed
246
247
248
249
250
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

251
252
253
254
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

255
256
257
258
259
260
261
262
263
264
265
266
267
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

268
269
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
270
271
272
	if (limit && end_pfn > limit)
		end_pfn = limit;

273
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
274
275
276
277
278

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
279
280
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
281
282
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
283

284
285
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
286
			if (!limit || (limit && limit > bdata->last_success))
287
				preferred = bdata->last_success - node_boot_start;
288
	}
Linus Torvalds's avatar
Linus Torvalds committed
289

290
	preferred = PFN_DOWN(ALIGN(preferred, align));
291
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
292
293
294
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
295
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
296
		unsigned long j;
297

298
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
299
		i = ALIGN(i, incr);
300
301
		if (i >= eidx)
			break;
302
		if (test_bit(i, node_bootmem_map)) {
303
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
304
			continue;
305
		}
Linus Torvalds's avatar
Linus Torvalds committed
306
307
308
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
309
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
310
311
312
313
314
315
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
316
317
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
318
319
	}

320
321
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
322
323
324
325
326
		goto restart_scan;
	}
	return NULL;

found:
327
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
328
329
330
331
332
333
334
335
336
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
337
		unsigned long offset, remaining_size;
338
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
339
		BUG_ON(offset > PAGE_SIZE);
340
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
341
342
343
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
344
345
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
346
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
347
348
		} else {
			remaining_size = size - remaining_size;
349
350
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
351
					   offset + node_boot_start);
352
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
353
354
355
356
357
358
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
359
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
360
361
362
363
364
	}

	/*
	 * Reserve the area now:
	 */
365
	for (i = start; i < start + areasize; i++)
366
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
367
368
369
370
371
372
373
374
			BUG();
	memset(ret, 0, size);
	return ret;
}

static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
{
	struct page *page;
375
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
376
377
378
379
380
381
382
383
384
385
	bootmem_data_t *bdata = pgdat->bdata;
	unsigned long i, count, total = 0;
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
386
387
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
Linus Torvalds's avatar
Linus Torvalds committed
388
389
390
391
392
393
394
	map = bdata->node_bootmem_map;
	/* Check physaddr is O(LOG2(BITS_PER_LONG)) page aligned */
	if (bdata->node_boot_start == 0 ||
	    ffs(bdata->node_boot_start) - PAGE_SHIFT > ffs(BITS_PER_LONG))
		gofast = 1;
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
395

Linus Torvalds's avatar
Linus Torvalds committed
396
		if (gofast && v == ~0UL) {
397
			int order;
Linus Torvalds's avatar
Linus Torvalds committed
398

399
			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
400
401
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
402
			__free_pages_bootmem(page, order);
Linus Torvalds's avatar
Linus Torvalds committed
403
404
405
406
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
407
408

			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
409
410
411
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
412
					__free_pages_bootmem(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
413
414
415
				}
			}
		} else {
416
			i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
417
		}
418
		pfn += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
419
420
421
422
423
424
425
426
427
	}
	total += count;

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
	count = 0;
428
429
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
	for (i = 0; i < idx; i++, page++) {
430
		__free_pages_bootmem(page, 0);
431
		count++;
Linus Torvalds's avatar
Linus Torvalds committed
432
433
434
435
436
437
438
	}
	total += count;
	bdata->node_bootmem_map = NULL;

	return total;
}

439
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
440
				unsigned long startpfn, unsigned long endpfn)
Linus Torvalds's avatar
Linus Torvalds committed
441
{
442
	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
Linus Torvalds's avatar
Linus Torvalds committed
443
444
}

445
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
446
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
447
{
448
449
450
451
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
452
		return -ENOMEM;
453
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
454
455

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
456
457
}

458
459
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
460
461
462
463
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

464
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
Linus Torvalds's avatar
Linus Torvalds committed
465
{
466
	register_page_bootmem_info_node(pgdat);
467
	return free_all_bootmem_core(pgdat);
Linus Torvalds's avatar
Linus Torvalds committed
468
469
}

470
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
471
472
473
{
	max_low_pfn = pages;
	min_low_pfn = start;
474
	return init_bootmem_core(NODE_DATA(0), start, 0, pages);
Linus Torvalds's avatar
Linus Torvalds committed
475
476
477
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
478
479
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
Linus Torvalds's avatar
Linus Torvalds committed
480
{
481
482
483
484
485
486
487
488
489
490
491
492
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
493
494
495
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

496
void __init free_bootmem(unsigned long addr, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
497
{
498
499
500
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
Linus Torvalds's avatar
Linus Torvalds committed
501
502
}

503
unsigned long __init free_all_bootmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
504
{
505
	return free_all_bootmem_core(NODE_DATA(0));
Linus Torvalds's avatar
Linus Torvalds committed
506
507
}

508
509
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
510
{
511
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
512
513
	void *ptr;

514
515
516
517
518
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
519
520
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
521

522
523
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
524
525
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
526

527
528
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
529
530
531
532
533
534
535
536
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

537

538
539
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
540
541
542
{
	void *ptr;

543
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
544
	if (ptr)
545
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
546

547
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
548
549
}

550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

581
582
583
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
584

585
586
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
587
{
588
	bootmem_data_t *bdata;
589
590
	void *ptr;

591
	list_for_each_entry(bdata, &bdata_list, list) {
592
593
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
594
595
596
		if (ptr)
			return ptr;
	}
597
598
599
600
601
602
603
604
605
606
607
608

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
609
610
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
611
}