bootmem.c 15.2 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

39
40
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;

41
42
43
44
45
46
47
48
49
50
51
52
53
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}

Linus Torvalds's avatar
Linus Torvalds committed
54
/* return the number of _pages_ that will be allocated for the boot bitmap */
55
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
56
57
58
59
60
61
62
63
64
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
65

66
67
68
/*
 * link bdata in order
 */
69
static void __init link_bootmem(bootmem_data_t *bdata)
70
71
{
	bootmem_data_t *ent;
72

73
74
75
76
77
78
79
80
81
82
83
84
85
86
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

Linus Torvalds's avatar
Linus Torvalds committed
87
88
89
/*
 * Called once to set up the allocator itself.
 */
90
static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds's avatar
Linus Torvalds committed
91
92
	unsigned long mapstart, unsigned long start, unsigned long end)
{
93
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
94

95
	mminit_validate_memmodel_limits(&start, &end);
96
97
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
98
	bdata->node_low_pfn = end;
99
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
100
101
102
103
104

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
105
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
106
107
108
109
110
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
				unsigned long startpfn, unsigned long endpfn)
{
	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
}

unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
{
	max_low_pfn = pages;
	min_low_pfn = start;
	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
}

static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
{
	struct page *page;
	unsigned long pfn;
	unsigned long i, count;
	unsigned long idx;
	unsigned long *map;
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
	map = bdata->node_bootmem_map;
	/*
	 * Check if we are aligned to BITS_PER_LONG pages.  If so, we might
	 * be able to free page orders of that size at once.
	 */
	if (!(pfn & (BITS_PER_LONG-1)))
		gofast = 1;

	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];

		if (gofast && v == ~0UL) {
			int order;

			page = pfn_to_page(pfn);
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
			__free_pages_bootmem(page, order);
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;

			page = pfn_to_page(pfn);
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
					__free_pages_bootmem(page, 0);
				}
			}
		} else {
			i += BITS_PER_LONG;
		}
		pfn += BITS_PER_LONG;
	}

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
	for (i = 0; i < idx; i++, page++)
		__free_pages_bootmem(page, 0);
	count += i;
	bdata->node_bootmem_map = NULL;

	return count;
}

unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
{
	register_page_bootmem_info_node(pgdat);
	return free_all_bootmem_core(pgdat->bdata);
}

unsigned long __init free_all_bootmem(void)
{
	return free_all_bootmem_core(NODE_DATA(0)->bdata);
}

static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
{
	unsigned long sidx, eidx;
	unsigned long i;

	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
		bdata->last_success = addr;

	/*
	 * Round up to index to the range.
	 */
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

void __init free_bootmem(unsigned long addr, unsigned long size)
{
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
}

Linus Torvalds's avatar
Linus Torvalds committed
251
252
253
254
255
/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
256
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
257
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
258
{
259
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
260
	unsigned long i;
261
262
263
264
265
266
267

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
268

Linus Torvalds's avatar
Linus Torvalds committed
269
	/*
270
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
271
	 */
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
298
	BUG_ON(!size);
299

300
301
302
303
304
305
306
307
308
309
310
311
312
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

313
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
314
315
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
316

317
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
318
319
320
321
322
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
323
	}
Linus Torvalds's avatar
Linus Torvalds committed
324
325
}

326
327
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
328
{
329
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
330

331
332
333
334
335
336
	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
		return -ENOMEM;
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	return 0;
}
337

338
339
340
341
342
343
#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
{
	bootmem_data_t *bdata;
	int ret;
Linus Torvalds's avatar
Linus Torvalds committed
344

345
346
347
348
	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
Linus Torvalds's avatar
Linus Torvalds committed
349
	}
350
351
352
353
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
354
}
355
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
370
371
372
static void * __init
alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
		unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
373
{
374
	unsigned long areasize, preferred;
375
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
376
	void *ret;
377
378
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
379

380
	if (!size) {
381
		printk("alloc_bootmem_core(): zero-sized request\n");
Linus Torvalds's avatar
Linus Torvalds committed
382
383
384
385
		BUG();
	}
	BUG_ON(align & (align-1));

386
387
388
389
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

390
391
392
393
394
395
396
397
398
399
400
401
402
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

403
404
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
405
406
407
	if (limit && end_pfn > limit)
		end_pfn = limit;

408
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
409
410
411
412
413

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
414
415
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
416
417
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
418

419
420
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
421
			if (!limit || (limit && limit > bdata->last_success))
422
				preferred = bdata->last_success - node_boot_start;
423
	}
Linus Torvalds's avatar
Linus Torvalds committed
424

425
	preferred = PFN_DOWN(ALIGN(preferred, align));
426
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
427
428
429
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
430
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
431
		unsigned long j;
432

433
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
434
		i = ALIGN(i, incr);
435
436
		if (i >= eidx)
			break;
437
		if (test_bit(i, node_bootmem_map)) {
438
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
439
			continue;
440
		}
Linus Torvalds's avatar
Linus Torvalds committed
441
442
443
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
444
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
445
446
447
448
449
450
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
451
452
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
453
454
	}

455
456
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
457
458
459
460
461
		goto restart_scan;
	}
	return NULL;

found:
462
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
463
464
465
466
467
468
469
470
471
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
472
		unsigned long offset, remaining_size;
473
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
474
		BUG_ON(offset > PAGE_SIZE);
475
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
476
477
478
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
479
480
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
481
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
482
483
		} else {
			remaining_size = size - remaining_size;
484
485
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
486
					   offset + node_boot_start);
487
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
488
489
490
491
492
493
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
494
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
495
496
497
498
499
	}

	/*
	 * Reserve the area now:
	 */
500
	for (i = start; i < start + areasize; i++)
501
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
502
503
504
505
506
			BUG();
	memset(ret, 0, size);
	return ret;
}

507
508
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
509
{
510
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
511
512
	void *ptr;

513
	list_for_each_entry(bdata, &bdata_list, list) {
514
		ptr = alloc_bootmem_core(bdata, size, align, goal, 0);
515
516
517
		if (ptr)
			return ptr;
	}
518
519
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
520

521
522
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
523
524
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
525

526
527
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
528
529
530
531
532
533
534
535
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

536
537
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
538
539
540
{
	void *ptr;

541
	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
542
	if (ptr)
543
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
544

545
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
546
547
}

548
549
550
551
552
553
554
555
556
557
558
559
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
560
561
	ptr = alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				limit);
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

579
580
581
582
583
584
585
586
587
588
589
590
void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
{
	void *ptr;

	ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
	if (ptr)
		return ptr;

	return __alloc_bootmem_nopanic(size, align, goal);
}

591
592
593
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
594

595
596
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
597
{
598
	bootmem_data_t *bdata;
599
600
	void *ptr;

601
	list_for_each_entry(bdata, &bdata_list, list) {
602
603
		ptr = alloc_bootmem_core(bdata, size, align, goal,
					ARCH_LOW_ADDRESS_LIMIT);
604
605
606
		if (ptr)
			return ptr;
	}
607
608
609
610
611
612
613
614
615
616
617
618

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
619
620
	return alloc_bootmem_core(pgdat->bdata, size, align, goal,
				ARCH_LOW_ADDRESS_LIMIT);
621
}