bootmem.c 14.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

39
40
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;

Linus Torvalds's avatar
Linus Torvalds committed
41
/* return the number of _pages_ that will be allocated for the boot bitmap */
42
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
43
44
45
46
47
48
49
50
51
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
52

53
54
55
/*
 * link bdata in order
 */
56
static void __init link_bootmem(bootmem_data_t *bdata)
57
58
{
	bootmem_data_t *ent;
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

74
75
76
77
78
79
80
81
82
83
84
85
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
Linus Torvalds's avatar
Linus Torvalds committed
86
87
88
89

/*
 * Called once to set up the allocator itself.
 */
90
static unsigned long __init init_bootmem_core(bootmem_data_t *bdata,
Linus Torvalds's avatar
Linus Torvalds committed
91
92
	unsigned long mapstart, unsigned long start, unsigned long end)
{
93
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
94

95
	mminit_validate_memmodel_limits(&start, &end);
96
97
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
98
	bdata->node_low_pfn = end;
99
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
100
101
102
103
104

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
105
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
106
107
108
109
110
111
112
113
114
115
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
116
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
117
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
118
{
119
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
120
	unsigned long i;
121
122
123
124
125
126
127

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
128

Linus Torvalds's avatar
Linus Torvalds committed
129
	/*
130
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
131
	 */
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
158
	BUG_ON(!size);
159

160
161
162
163
164
165
166
167
168
169
170
171
172
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

173
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
174
175
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
176

177
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
178
179
180
181
182
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
183
	}
Linus Torvalds's avatar
Linus Torvalds committed
184
185
}

186
187
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
188
{
189
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
190
	unsigned long i;
191

192
193
194
195
196
197
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
198
199
200
201
202
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

203
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
Linus Torvalds's avatar
Linus Torvalds committed
204
205
206
		bdata->last_success = addr;

	/*
207
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
208
	 */
209
210
211
212
213
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

214
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
215
216
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
237
void * __init
Linus Torvalds's avatar
Linus Torvalds committed
238
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
239
	      unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
240
{
241
	unsigned long areasize, preferred;
242
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
243
	void *ret;
244
245
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
246

247
	if (!size) {
Linus Torvalds's avatar
Linus Torvalds committed
248
249
250
251
252
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

253
254
255
256
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

257
258
259
260
261
262
263
264
265
266
267
268
269
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

270
271
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
272
273
274
	if (limit && end_pfn > limit)
		end_pfn = limit;

275
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
276
277
278
279
280

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
281
282
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
283
284
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
285

286
287
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
288
			if (!limit || (limit && limit > bdata->last_success))
289
				preferred = bdata->last_success - node_boot_start;
290
	}
Linus Torvalds's avatar
Linus Torvalds committed
291

292
	preferred = PFN_DOWN(ALIGN(preferred, align));
293
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
294
295
296
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
297
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
298
		unsigned long j;
299

300
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
301
		i = ALIGN(i, incr);
302
303
		if (i >= eidx)
			break;
304
		if (test_bit(i, node_bootmem_map)) {
305
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
306
			continue;
307
		}
Linus Torvalds's avatar
Linus Torvalds committed
308
309
310
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
311
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
312
313
314
315
316
317
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
318
319
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
320
321
	}

322
323
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
324
325
326
327
328
		goto restart_scan;
	}
	return NULL;

found:
329
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
330
331
332
333
334
335
336
337
338
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
339
		unsigned long offset, remaining_size;
340
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
341
		BUG_ON(offset > PAGE_SIZE);
342
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
343
344
345
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
346
347
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
348
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
349
350
		} else {
			remaining_size = size - remaining_size;
351
352
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
353
					   offset + node_boot_start);
354
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
355
356
357
358
359
360
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
361
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
362
363
364
365
366
	}

	/*
	 * Reserve the area now:
	 */
367
	for (i = start; i < start + areasize; i++)
368
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
369
370
371
372
373
			BUG();
	memset(ret, 0, size);
	return ret;
}

374
static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
Linus Torvalds's avatar
Linus Torvalds committed
375
376
{
	struct page *page;
377
	unsigned long pfn;
378
	unsigned long i, count;
Linus Torvalds's avatar
Linus Torvalds committed
379
380
381
382
383
384
385
386
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
387
388
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
Linus Torvalds's avatar
Linus Torvalds committed
389
	map = bdata->node_bootmem_map;
390
391
392
393
394
	/*
	 * Check if we are aligned to BITS_PER_LONG pages.  If so, we might
	 * be able to free page orders of that size at once.
	 */
	if (!(pfn & (BITS_PER_LONG-1)))
Linus Torvalds's avatar
Linus Torvalds committed
395
		gofast = 1;
396

Linus Torvalds's avatar
Linus Torvalds committed
397
398
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
399

Linus Torvalds's avatar
Linus Torvalds committed
400
		if (gofast && v == ~0UL) {
401
			int order;
Linus Torvalds's avatar
Linus Torvalds committed
402

403
			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
404
405
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
406
			__free_pages_bootmem(page, order);
Linus Torvalds's avatar
Linus Torvalds committed
407
408
409
410
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
411
412

			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
413
414
415
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
416
					__free_pages_bootmem(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
417
418
419
				}
			}
		} else {
420
			i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
421
		}
422
		pfn += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
423
424
425
426
427
428
429
	}

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
430
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
431
	for (i = 0; i < idx; i++, page++)
432
		__free_pages_bootmem(page, 0);
433
	count += i;
Linus Torvalds's avatar
Linus Torvalds committed
434
435
	bdata->node_bootmem_map = NULL;

436
	return count;
Linus Torvalds's avatar
Linus Torvalds committed
437
438
}

439
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
440
				unsigned long startpfn, unsigned long endpfn)
Linus Torvalds's avatar
Linus Torvalds committed
441
{
442
	return init_bootmem_core(pgdat->bdata, freepfn, startpfn, endpfn);
Linus Torvalds's avatar
Linus Torvalds committed
443
444
}

445
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
446
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
447
{
448
449
450
451
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
452
		return -ENOMEM;
453
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
454
455

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
456
457
}

458
459
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
460
461
462
463
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

464
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
Linus Torvalds's avatar
Linus Torvalds committed
465
{
466
	register_page_bootmem_info_node(pgdat);
467
	return free_all_bootmem_core(pgdat->bdata);
Linus Torvalds's avatar
Linus Torvalds committed
468
469
}

470
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
471
472
473
{
	max_low_pfn = pages;
	min_low_pfn = start;
474
	return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
Linus Torvalds's avatar
Linus Torvalds committed
475
476
477
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
478
479
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
Linus Torvalds's avatar
Linus Torvalds committed
480
{
481
482
483
484
485
486
487
488
489
490
491
492
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
493
494
495
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

496
void __init free_bootmem(unsigned long addr, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
497
{
498
499
500
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
Linus Torvalds's avatar
Linus Torvalds committed
501
502
}

503
unsigned long __init free_all_bootmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
504
{
505
	return free_all_bootmem_core(NODE_DATA(0)->bdata);
Linus Torvalds's avatar
Linus Torvalds committed
506
507
}

508
509
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
510
{
511
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
512
513
	void *ptr;

514
515
516
517
518
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
519
520
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
521

522
523
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
524
525
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
526

527
528
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
529
530
531
532
533
534
535
536
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

537

538
539
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
540
541
542
{
	void *ptr;

543
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
544
	if (ptr)
545
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
546

547
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
548
549
}

550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

581
582
583
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
584

585
586
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
587
{
588
	bootmem_data_t *bdata;
589
590
	void *ptr;

591
	list_for_each_entry(bdata, &bdata_list, list) {
592
593
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
594
595
596
		if (ptr)
			return ptr;
	}
597
598
599
600
601
602
603
604
605
606
607
608

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
609
610
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
611
}