bootmem.c 15 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
11
/*
 *  linux/mm/bootmem.c
 *
 *  Copyright (C) 1999 Ingo Molnar
 *  Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
 *
 *  simple boot-time physical memory area allocator and
 *  free memory collector. It's used to deal with reserved
 *  system memory and memory holes as well.
 */
#include <linux/init.h>
12
#include <linux/pfn.h>
Linus Torvalds's avatar
Linus Torvalds committed
13
14
#include <linux/bootmem.h>
#include <linux/module.h>
15
16

#include <asm/bug.h>
Linus Torvalds's avatar
Linus Torvalds committed
17
#include <asm/io.h>
18
#include <asm/processor.h>
19

Linus Torvalds's avatar
Linus Torvalds committed
20
21
22
23
24
25
26
27
28
29
#include "internal.h"

/*
 * Access to this subsystem has to be serialized externally. (this is
 * true for the boot process anyway)
 */
unsigned long max_low_pfn;
unsigned long min_low_pfn;
unsigned long max_pfn;

30
static LIST_HEAD(bdata_list);
31
32
33
34
35
36
37
38
#ifdef CONFIG_CRASH_DUMP
/*
 * If we have booted due to a crash, max_pfn will be a very low value. We need
 * to know the amount of memory that the previous kernel used.
 */
unsigned long saved_max_pfn;
#endif

39
40
bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;

Linus Torvalds's avatar
Linus Torvalds committed
41
/* return the number of _pages_ that will be allocated for the boot bitmap */
42
unsigned long __init bootmem_bootmap_pages(unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
43
44
45
46
47
48
49
50
51
{
	unsigned long mapsize;

	mapsize = (pages+7)/8;
	mapsize = (mapsize + ~PAGE_MASK) & PAGE_MASK;
	mapsize >>= PAGE_SHIFT;

	return mapsize;
}
52

53
54
55
/*
 * link bdata in order
 */
56
static void __init link_bootmem(bootmem_data_t *bdata)
57
58
{
	bootmem_data_t *ent;
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
	if (list_empty(&bdata_list)) {
		list_add(&bdata->list, &bdata_list);
		return;
	}
	/* insert in order */
	list_for_each_entry(ent, &bdata_list, list) {
		if (bdata->node_boot_start < ent->node_boot_start) {
			list_add_tail(&bdata->list, &ent->list);
			return;
		}
	}
	list_add_tail(&bdata->list, &bdata_list);
}

74
75
76
77
78
79
80
81
82
83
84
85
/*
 * Given an initialised bdata, it returns the size of the boot bitmap
 */
static unsigned long __init get_mapsize(bootmem_data_t *bdata)
{
	unsigned long mapsize;
	unsigned long start = PFN_DOWN(bdata->node_boot_start);
	unsigned long end = bdata->node_low_pfn;

	mapsize = ((end - start) + 7) / 8;
	return ALIGN(mapsize, sizeof(long));
}
Linus Torvalds's avatar
Linus Torvalds committed
86
87
88
89

/*
 * Called once to set up the allocator itself.
 */
90
static unsigned long __init init_bootmem_core(pg_data_t *pgdat,
Linus Torvalds's avatar
Linus Torvalds committed
91
92
93
	unsigned long mapstart, unsigned long start, unsigned long end)
{
	bootmem_data_t *bdata = pgdat->bdata;
94
	unsigned long mapsize;
Linus Torvalds's avatar
Linus Torvalds committed
95

96
	mminit_validate_memmodel_limits(&start, &end);
97
98
	bdata->node_bootmem_map = phys_to_virt(PFN_PHYS(mapstart));
	bdata->node_boot_start = PFN_PHYS(start);
Linus Torvalds's avatar
Linus Torvalds committed
99
	bdata->node_low_pfn = end;
100
	link_bootmem(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
101
102
103
104
105

	/*
	 * Initially all pages are reserved - setup_arch() has to
	 * register free RAM areas explicitly.
	 */
106
	mapsize = get_mapsize(bdata);
Linus Torvalds's avatar
Linus Torvalds committed
107
108
109
110
111
112
113
114
115
116
	memset(bdata->node_bootmem_map, 0xff, mapsize);

	return mapsize;
}

/*
 * Marks a particular physical memory range as unallocatable. Usable RAM
 * might be used for boot-time allocations - or it might get added
 * to the free page pool later on.
 */
117
static int __init can_reserve_bootmem_core(bootmem_data_t *bdata,
118
			unsigned long addr, unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
119
{
120
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
121
	unsigned long i;
122
123
124
125
126
127
128

	BUG_ON(!size);

	/* out of range, don't hold other */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return 0;
129

Linus Torvalds's avatar
Linus Torvalds committed
130
	/*
131
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
132
	 */
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

	eidx = PFN_UP(addr + size - bdata->node_boot_start);
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);

	for (i = sidx; i < eidx; i++) {
		if (test_bit(i, bdata->node_bootmem_map)) {
			if (flags & BOOTMEM_EXCLUSIVE)
				return -EBUSY;
		}
	}

	return 0;

}

static void __init reserve_bootmem_core(bootmem_data_t *bdata,
			unsigned long addr, unsigned long size, int flags)
{
	unsigned long sidx, eidx;
	unsigned long i;

Linus Torvalds's avatar
Linus Torvalds committed
159
	BUG_ON(!size);
160

161
162
163
164
165
166
167
168
169
170
171
172
173
	/* out of range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;

	/*
	 * Round up to index to the range.
	 */
	if (addr > bdata->node_boot_start)
		sidx= PFN_DOWN(addr - bdata->node_boot_start);
	else
		sidx = 0;

174
	eidx = PFN_UP(addr + size - bdata->node_boot_start);
175
176
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
177

178
	for (i = sidx; i < eidx; i++) {
Linus Torvalds's avatar
Linus Torvalds committed
179
180
181
182
183
		if (test_and_set_bit(i, bdata->node_bootmem_map)) {
#ifdef CONFIG_DEBUG_BOOTMEM
			printk("hm, page %08lx reserved twice.\n", i*PAGE_SIZE);
#endif
		}
184
	}
Linus Torvalds's avatar
Linus Torvalds committed
185
186
}

187
188
static void __init free_bootmem_core(bootmem_data_t *bdata, unsigned long addr,
				     unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
189
{
190
	unsigned long sidx, eidx;
Linus Torvalds's avatar
Linus Torvalds committed
191
	unsigned long i;
192

193
194
195
196
197
198
	BUG_ON(!size);

	/* out range */
	if (addr + size < bdata->node_boot_start ||
		PFN_DOWN(addr) > bdata->node_low_pfn)
		return;
Linus Torvalds's avatar
Linus Torvalds committed
199
200
201
202
203
	/*
	 * round down end of usable mem, partially free pages are
	 * considered reserved.
	 */

204
	if (addr >= bdata->node_boot_start && addr < bdata->last_success)
Linus Torvalds's avatar
Linus Torvalds committed
205
206
207
		bdata->last_success = addr;

	/*
208
	 * Round up to index to the range.
Linus Torvalds's avatar
Linus Torvalds committed
209
	 */
210
211
212
213
214
	if (PFN_UP(addr) > PFN_DOWN(bdata->node_boot_start))
		sidx = PFN_UP(addr) - PFN_DOWN(bdata->node_boot_start);
	else
		sidx = 0;

215
	eidx = PFN_DOWN(addr + size - bdata->node_boot_start);
216
217
	if (eidx > bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start))
		eidx = bdata->node_low_pfn - PFN_DOWN(bdata->node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237

	for (i = sidx; i < eidx; i++) {
		if (unlikely(!test_and_clear_bit(i, bdata->node_bootmem_map)))
			BUG();
	}
}

/*
 * We 'merge' subsequent allocations to save space. We might 'lose'
 * some fraction of a page if allocations cannot be satisfied due to
 * size constraints on boxes where there is physical RAM space
 * fragmentation - in these cases (mostly large memory boxes) this
 * is not a problem.
 *
 * On low memory boxes we get it right in 100% of the cases.
 *
 * alignment has to be a power of 2 value.
 *
 * NOTE:  This function is _not_ reentrant.
 */
238
void * __init
Linus Torvalds's avatar
Linus Torvalds committed
239
__alloc_bootmem_core(struct bootmem_data *bdata, unsigned long size,
240
	      unsigned long align, unsigned long goal, unsigned long limit)
Linus Torvalds's avatar
Linus Torvalds committed
241
{
242
	unsigned long areasize, preferred;
243
	unsigned long i, start = 0, incr, eidx, end_pfn;
Linus Torvalds's avatar
Linus Torvalds committed
244
	void *ret;
245
246
	unsigned long node_boot_start;
	void *node_bootmem_map;
Linus Torvalds's avatar
Linus Torvalds committed
247

248
	if (!size) {
Linus Torvalds's avatar
Linus Torvalds committed
249
250
251
252
253
		printk("__alloc_bootmem_core(): zero-sized request\n");
		BUG();
	}
	BUG_ON(align & (align-1));

254
255
256
257
	/* on nodes without memory - bootmem_map is NULL */
	if (!bdata->node_bootmem_map)
		return NULL;

258
259
260
261
262
263
264
265
266
267
268
269
270
	/* bdata->node_boot_start is supposed to be (12+6)bits alignment on x86_64 ? */
	node_boot_start = bdata->node_boot_start;
	node_bootmem_map = bdata->node_bootmem_map;
	if (align) {
		node_boot_start = ALIGN(bdata->node_boot_start, align);
		if (node_boot_start > bdata->node_boot_start)
			node_bootmem_map = (unsigned long *)bdata->node_bootmem_map +
			    PFN_DOWN(node_boot_start - bdata->node_boot_start)/BITS_PER_LONG;
	}

	if (limit && node_boot_start >= limit)
		return NULL;

271
272
	end_pfn = bdata->node_low_pfn;
	limit = PFN_DOWN(limit);
273
274
275
	if (limit && end_pfn > limit)
		end_pfn = limit;

276
	eidx = end_pfn - PFN_DOWN(node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
277
278
279
280
281

	/*
	 * We try to allocate bootmem pages above 'goal'
	 * first, then we try to allocate lower pages.
	 */
282
283
	preferred = 0;
	if (goal && PFN_DOWN(goal) < end_pfn) {
284
285
		if (goal > node_boot_start)
			preferred = goal - node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
286

287
288
		if (bdata->last_success > node_boot_start &&
			bdata->last_success - node_boot_start >= preferred)
289
			if (!limit || (limit && limit > bdata->last_success))
290
				preferred = bdata->last_success - node_boot_start;
291
	}
Linus Torvalds's avatar
Linus Torvalds committed
292

293
	preferred = PFN_DOWN(ALIGN(preferred, align));
294
	areasize = (size + PAGE_SIZE-1) / PAGE_SIZE;
Linus Torvalds's avatar
Linus Torvalds committed
295
296
297
	incr = align >> PAGE_SHIFT ? : 1;

restart_scan:
298
	for (i = preferred; i < eidx;) {
Linus Torvalds's avatar
Linus Torvalds committed
299
		unsigned long j;
300

301
		i = find_next_zero_bit(node_bootmem_map, eidx, i);
Linus Torvalds's avatar
Linus Torvalds committed
302
		i = ALIGN(i, incr);
303
304
		if (i >= eidx)
			break;
305
		if (test_bit(i, node_bootmem_map)) {
306
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
307
			continue;
308
		}
Linus Torvalds's avatar
Linus Torvalds committed
309
310
311
		for (j = i + 1; j < i + areasize; ++j) {
			if (j >= eidx)
				goto fail_block;
312
			if (test_bit(j, node_bootmem_map))
Linus Torvalds's avatar
Linus Torvalds committed
313
314
315
316
317
318
				goto fail_block;
		}
		start = i;
		goto found;
	fail_block:
		i = ALIGN(j, incr);
319
320
		if (i == j)
			i += incr;
Linus Torvalds's avatar
Linus Torvalds committed
321
322
	}

323
324
	if (preferred > 0) {
		preferred = 0;
Linus Torvalds's avatar
Linus Torvalds committed
325
326
327
328
329
		goto restart_scan;
	}
	return NULL;

found:
330
	bdata->last_success = PFN_PHYS(start) + node_boot_start;
Linus Torvalds's avatar
Linus Torvalds committed
331
332
333
334
335
336
337
338
339
	BUG_ON(start >= eidx);

	/*
	 * Is the next page of the previous allocation-end the start
	 * of this allocation's buffer? If yes then we can 'merge'
	 * the previous partial page with this allocation.
	 */
	if (align < PAGE_SIZE &&
	    bdata->last_offset && bdata->last_pos+1 == start) {
340
		unsigned long offset, remaining_size;
341
		offset = ALIGN(bdata->last_offset, align);
Linus Torvalds's avatar
Linus Torvalds committed
342
		BUG_ON(offset > PAGE_SIZE);
343
		remaining_size = PAGE_SIZE - offset;
Linus Torvalds's avatar
Linus Torvalds committed
344
345
346
		if (size < remaining_size) {
			areasize = 0;
			/* last_pos unchanged */
347
348
			bdata->last_offset = offset + size;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
349
					   offset + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
350
351
		} else {
			remaining_size = size - remaining_size;
352
353
			areasize = (remaining_size + PAGE_SIZE-1) / PAGE_SIZE;
			ret = phys_to_virt(bdata->last_pos * PAGE_SIZE +
354
					   offset + node_boot_start);
355
			bdata->last_pos = start + areasize - 1;
Linus Torvalds's avatar
Linus Torvalds committed
356
357
358
359
360
361
			bdata->last_offset = remaining_size;
		}
		bdata->last_offset &= ~PAGE_MASK;
	} else {
		bdata->last_pos = start + areasize - 1;
		bdata->last_offset = size & ~PAGE_MASK;
362
		ret = phys_to_virt(start * PAGE_SIZE + node_boot_start);
Linus Torvalds's avatar
Linus Torvalds committed
363
364
365
366
367
	}

	/*
	 * Reserve the area now:
	 */
368
	for (i = start; i < start + areasize; i++)
369
		if (unlikely(test_and_set_bit(i, node_bootmem_map)))
Linus Torvalds's avatar
Linus Torvalds committed
370
371
372
373
374
375
376
377
			BUG();
	memset(ret, 0, size);
	return ret;
}

static unsigned long __init free_all_bootmem_core(pg_data_t *pgdat)
{
	struct page *page;
378
	unsigned long pfn;
Linus Torvalds's avatar
Linus Torvalds committed
379
	bootmem_data_t *bdata = pgdat->bdata;
380
	unsigned long i, count;
Linus Torvalds's avatar
Linus Torvalds committed
381
382
383
384
385
386
387
388
	unsigned long idx;
	unsigned long *map; 
	int gofast = 0;

	BUG_ON(!bdata->node_bootmem_map);

	count = 0;
	/* first extant page of the node */
389
390
	pfn = PFN_DOWN(bdata->node_boot_start);
	idx = bdata->node_low_pfn - pfn;
Linus Torvalds's avatar
Linus Torvalds committed
391
	map = bdata->node_bootmem_map;
392
393
394
395
396
	/*
	 * Check if we are aligned to BITS_PER_LONG pages.  If so, we might
	 * be able to free page orders of that size at once.
	 */
	if (!(pfn & (BITS_PER_LONG-1)))
Linus Torvalds's avatar
Linus Torvalds committed
397
		gofast = 1;
398

Linus Torvalds's avatar
Linus Torvalds committed
399
400
	for (i = 0; i < idx; ) {
		unsigned long v = ~map[i / BITS_PER_LONG];
401

Linus Torvalds's avatar
Linus Torvalds committed
402
		if (gofast && v == ~0UL) {
403
			int order;
Linus Torvalds's avatar
Linus Torvalds committed
404

405
			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
406
407
			count += BITS_PER_LONG;
			order = ffs(BITS_PER_LONG) - 1;
408
			__free_pages_bootmem(page, order);
Linus Torvalds's avatar
Linus Torvalds committed
409
410
411
412
			i += BITS_PER_LONG;
			page += BITS_PER_LONG;
		} else if (v) {
			unsigned long m;
413
414

			page = pfn_to_page(pfn);
Linus Torvalds's avatar
Linus Torvalds committed
415
416
417
			for (m = 1; m && i < idx; m<<=1, page++, i++) {
				if (v & m) {
					count++;
418
					__free_pages_bootmem(page, 0);
Linus Torvalds's avatar
Linus Torvalds committed
419
420
421
				}
			}
		} else {
422
			i += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
423
		}
424
		pfn += BITS_PER_LONG;
Linus Torvalds's avatar
Linus Torvalds committed
425
426
427
428
429
430
431
	}

	/*
	 * Now free the allocator bitmap itself, it's not
	 * needed anymore:
	 */
	page = virt_to_page(bdata->node_bootmem_map);
432
	idx = (get_mapsize(bdata) + PAGE_SIZE-1) >> PAGE_SHIFT;
433
	for (i = 0; i < idx; i++, page++)
434
		__free_pages_bootmem(page, 0);
435
	count += i;
Linus Torvalds's avatar
Linus Torvalds committed
436
437
	bdata->node_bootmem_map = NULL;

438
	return count;
Linus Torvalds's avatar
Linus Torvalds committed
439
440
}

441
unsigned long __init init_bootmem_node(pg_data_t *pgdat, unsigned long freepfn,
442
				unsigned long startpfn, unsigned long endpfn)
Linus Torvalds's avatar
Linus Torvalds committed
443
{
444
	return init_bootmem_core(pgdat, freepfn, startpfn, endpfn);
Linus Torvalds's avatar
Linus Torvalds committed
445
446
}

447
int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
448
				 unsigned long size, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
449
{
450
451
452
453
	int ret;

	ret = can_reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
	if (ret < 0)
454
		return -ENOMEM;
455
	reserve_bootmem_core(pgdat->bdata, physaddr, size, flags);
456
457

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
458
459
}

460
461
void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
			      unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
462
463
464
465
{
	free_bootmem_core(pgdat->bdata, physaddr, size);
}

466
unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
Linus Torvalds's avatar
Linus Torvalds committed
467
{
468
	register_page_bootmem_info_node(pgdat);
469
	return free_all_bootmem_core(pgdat);
Linus Torvalds's avatar
Linus Torvalds committed
470
471
}

472
unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
Linus Torvalds's avatar
Linus Torvalds committed
473
474
475
{
	max_low_pfn = pages;
	min_low_pfn = start;
476
	return init_bootmem_core(NODE_DATA(0), start, 0, pages);
Linus Torvalds's avatar
Linus Torvalds committed
477
478
479
}

#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
480
481
int __init reserve_bootmem(unsigned long addr, unsigned long size,
			    int flags)
Linus Torvalds's avatar
Linus Torvalds committed
482
{
483
484
485
486
487
488
489
490
491
492
493
494
	bootmem_data_t *bdata;
	int ret;

	list_for_each_entry(bdata, &bdata_list, list) {
		ret = can_reserve_bootmem_core(bdata, addr, size, flags);
		if (ret < 0)
			return ret;
	}
	list_for_each_entry(bdata, &bdata_list, list)
		reserve_bootmem_core(bdata, addr, size, flags);

	return 0;
Linus Torvalds's avatar
Linus Torvalds committed
495
496
497
}
#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */

498
void __init free_bootmem(unsigned long addr, unsigned long size)
Linus Torvalds's avatar
Linus Torvalds committed
499
{
500
501
502
	bootmem_data_t *bdata;
	list_for_each_entry(bdata, &bdata_list, list)
		free_bootmem_core(bdata, addr, size);
Linus Torvalds's avatar
Linus Torvalds committed
503
504
}

505
unsigned long __init free_all_bootmem(void)
Linus Torvalds's avatar
Linus Torvalds committed
506
{
507
	return free_all_bootmem_core(NODE_DATA(0));
Linus Torvalds's avatar
Linus Torvalds committed
508
509
}

510
511
void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
				      unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
512
{
513
	bootmem_data_t *bdata;
Linus Torvalds's avatar
Linus Torvalds committed
514
515
	void *ptr;

516
517
518
519
520
	list_for_each_entry(bdata, &bdata_list, list) {
		ptr = __alloc_bootmem_core(bdata, size, align, goal, 0);
		if (ptr)
			return ptr;
	}
521
522
	return NULL;
}
Linus Torvalds's avatar
Linus Torvalds committed
523

524
525
void * __init __alloc_bootmem(unsigned long size, unsigned long align,
			      unsigned long goal)
526
527
{
	void *mem = __alloc_bootmem_nopanic(size,align,goal);
528

529
530
	if (mem)
		return mem;
Linus Torvalds's avatar
Linus Torvalds committed
531
532
533
534
535
536
537
538
	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of memory");
	return NULL;
}

539

540
541
void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
				   unsigned long align, unsigned long goal)
Linus Torvalds's avatar
Linus Torvalds committed
542
543
544
{
	void *ptr;

545
	ptr = __alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
Linus Torvalds's avatar
Linus Torvalds committed
546
	if (ptr)
547
		return ptr;
Linus Torvalds's avatar
Linus Torvalds committed
548

549
	return __alloc_bootmem(size, align, goal);
Linus Torvalds's avatar
Linus Torvalds committed
550
551
}

552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
#ifdef CONFIG_SPARSEMEM
void * __init alloc_bootmem_section(unsigned long size,
				    unsigned long section_nr)
{
	void *ptr;
	unsigned long limit, goal, start_nr, end_nr, pfn;
	struct pglist_data *pgdat;

	pfn = section_nr_to_pfn(section_nr);
	goal = PFN_PHYS(pfn);
	limit = PFN_PHYS(section_nr_to_pfn(section_nr + 1)) - 1;
	pgdat = NODE_DATA(early_pfn_to_nid(pfn));
	ptr = __alloc_bootmem_core(pgdat->bdata, size, SMP_CACHE_BYTES, goal,
				   limit);

	if (!ptr)
		return NULL;

	start_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr)));
	end_nr = pfn_to_section_nr(PFN_DOWN(__pa(ptr) + size));
	if (start_nr != section_nr || end_nr != section_nr) {
		printk(KERN_WARNING "alloc_bootmem failed on section %ld.\n",
		       section_nr);
		free_bootmem_core(pgdat->bdata, __pa(ptr), size);
		ptr = NULL;
	}

	return ptr;
}
#endif

583
584
585
#ifndef ARCH_LOW_ADDRESS_LIMIT
#define ARCH_LOW_ADDRESS_LIMIT	0xffffffffUL
#endif
586

587
588
void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
				  unsigned long goal)
589
{
590
	bootmem_data_t *bdata;
591
592
	void *ptr;

593
	list_for_each_entry(bdata, &bdata_list, list) {
594
595
		ptr = __alloc_bootmem_core(bdata, size, align, goal,
						ARCH_LOW_ADDRESS_LIMIT);
596
597
598
		if (ptr)
			return ptr;
	}
599
600
601
602
603
604
605
606
607
608
609
610

	/*
	 * Whoops, we cannot satisfy the allocation request.
	 */
	printk(KERN_ALERT "low bootmem alloc of %lu bytes failed!\n", size);
	panic("Out of low memory");
	return NULL;
}

void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
				       unsigned long align, unsigned long goal)
{
611
612
	return __alloc_bootmem_core(pgdat->bdata, size, align, goal,
				    ARCH_LOW_ADDRESS_LIMIT);
613
}